{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Torch Study\n",
    "## Three main features of PyTorch\n",
    " - Tensor: similar to numpy.array but can run on GPUs\n",
    " - Autograd: automatic differentiation for all operations on Tensors\n",
    " - NN: nn.module, framework to build neural network easily\n",
    "\n",
    "## Materials\n",
    " - [PyTorch Documentation](https://pytorch.org/docs/stable/index.html)\n",
    " - [PyTorch Examples](https://github.com/jcjohnson/pytorch-examples)\n",
    "\n",
    "## Examples\n",
    " - 3 Layers Neural Network\n",
    "   - Input Layer: 1000 neurons\n",
    "   - Hidden Layer: 100 neurons\n",
    "      - ReLU Activation Function\n",
    "   - Output Layer: 10 neurons\n",
    "- Training Data: 100 samples\n",
    "   - Learning Rate: 1e-6\n",
    "   - Training Iterations: 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 4578347.941021423\n",
      "1 2471386.0550409984\n",
      "2 1865985.8169652259\n",
      "3 3518134.9923215457\n",
      "4 16376692.467875663\n",
      "5 93889750.51186724\n",
      "6 474853966.242233\n",
      "7 993316544.0218148\n",
      "8 38490045.170285694\n",
      "9 17028018.9979011\n",
      "10 10448400.295304652\n",
      "11 7196131.101538438\n",
      "12 5282199.816150585\n",
      "13 4033626.3417542824\n",
      "14 3167301.049943808\n",
      "15 2538802.706842456\n",
      "16 2066920.0160529031\n",
      "17 1703388.0161938835\n",
      "18 1417767.3379489803\n",
      "19 1189900.9767300896\n",
      "20 1005686.9477772282\n",
      "21 855074.441256687\n",
      "22 730853.337463073\n",
      "23 627655.512285466\n",
      "24 541338.3082262024\n",
      "25 468750.69251051976\n",
      "26 407510.7111747361\n",
      "27 355578.9804186145\n",
      "28 311362.79884176777\n",
      "29 273606.5310706841\n",
      "30 241234.08519978856\n",
      "31 213426.0353554436\n",
      "32 189469.81166678556\n",
      "33 168766.03131946674\n",
      "34 150850.84380896413\n",
      "35 135312.27452102417\n",
      "36 121808.48789865169\n",
      "37 110046.90297558189\n",
      "38 99775.54929768015\n",
      "39 90793.07504443655\n",
      "40 82917.42556246182\n",
      "41 75991.90623918288\n",
      "42 69892.15909631057\n",
      "43 64508.966622447435\n",
      "44 59742.53656215685\n",
      "45 55509.59114687612\n",
      "46 51740.3089052528\n",
      "47 48375.839649100286\n",
      "48 45365.654468402856\n",
      "49 42663.519276498686\n",
      "50 40232.934298856\n",
      "51 38038.179744938\n",
      "52 36051.002384229214\n",
      "53 34245.532236748135\n",
      "54 32600.6374981621\n",
      "55 31097.594090268856\n",
      "56 29719.714830247616\n",
      "57 28453.005072838354\n",
      "58 27284.819426544385\n",
      "59 26205.186925368387\n",
      "60 25203.70467089275\n",
      "61 24272.00740596053\n",
      "62 23403.474049063436\n",
      "63 22592.154120719137\n",
      "64 21832.08704311895\n",
      "65 21118.678689288674\n",
      "66 20446.726427420734\n",
      "67 19813.048195601383\n",
      "68 19213.511225653016\n",
      "69 18645.585932455888\n",
      "70 18106.751328635626\n",
      "71 17594.851461782353\n",
      "72 17107.09347190864\n",
      "73 16642.019491776617\n",
      "74 16198.16511951957\n",
      "75 15773.659669097131\n",
      "76 15367.696310939826\n",
      "77 14978.631503695642\n",
      "78 14606.05097348859\n",
      "79 14248.426066185195\n",
      "80 13904.482119829985\n",
      "81 13573.945953089325\n",
      "82 13255.560258178253\n",
      "83 12948.75763147876\n",
      "84 12653.025089855684\n",
      "85 12367.614355241802\n",
      "86 12091.823121058727\n",
      "87 11825.405637632713\n",
      "88 11567.924373707363\n",
      "89 11318.900901199728\n",
      "90 11077.883775905495\n",
      "91 10844.614839662549\n",
      "92 10618.780043509905\n",
      "93 10399.888645090428\n",
      "94 10187.771340811503\n",
      "95 9982.114592843558\n",
      "96 9782.769943312958\n",
      "97 9589.186357204508\n",
      "98 9401.328933014258\n",
      "99 9219.10317766757\n",
      "100 9042.162250461872\n",
      "101 8870.157602804113\n",
      "102 8702.980943981087\n",
      "103 8540.624153243425\n",
      "104 8382.690444680682\n",
      "105 8229.014513991164\n",
      "106 8079.568517453535\n",
      "107 7934.113061425326\n",
      "108 7792.529077100533\n",
      "109 7654.680827711782\n",
      "110 7520.616517808542\n",
      "111 7389.8867408221795\n",
      "112 7262.523303804728\n",
      "113 7138.462818625586\n",
      "114 7017.5961976563485\n",
      "115 6900.063390660098\n",
      "116 6785.65154376457\n",
      "117 6674.07824272328\n",
      "118 6565.259801791673\n",
      "119 6459.106332152537\n",
      "120 6355.569427576687\n",
      "121 6254.500833906674\n",
      "122 6155.810909396155\n",
      "123 6059.518953770197\n",
      "124 5965.447871899782\n",
      "125 5873.54685556217\n",
      "126 5783.847715804149\n",
      "127 5696.142407400813\n",
      "128 5610.4041052261\n",
      "129 5526.658143874046\n",
      "130 5445.078163047939\n",
      "131 5365.995612599019\n",
      "132 5288.686613104694\n",
      "133 5213.040992350126\n",
      "134 5139.0451907333345\n",
      "135 5066.716869898949\n",
      "136 4995.900930626611\n",
      "137 4926.585589570022\n",
      "138 4858.754657459445\n",
      "139 4792.366102555351\n",
      "140 4727.305982333892\n",
      "141 4663.633851844161\n",
      "142 4601.250532399821\n",
      "143 4540.12203483145\n",
      "144 4480.275742666163\n",
      "145 4421.602900054823\n",
      "146 4364.115133373657\n",
      "147 4307.785467735633\n",
      "148 4252.544065440599\n",
      "149 4198.363869489594\n",
      "150 4145.269270550927\n",
      "151 4093.1857611985392\n",
      "152 4042.1202285836803\n",
      "153 3992.0293294792627\n",
      "154 3942.900570350448\n",
      "155 3894.7209672604736\n",
      "156 3847.438225404748\n",
      "157 3800.9954162095287\n",
      "158 3755.460364750606\n",
      "159 3710.738522183347\n",
      "160 3666.808739152848\n",
      "161 3623.712677090929\n",
      "162 3581.358526254711\n",
      "163 3539.7956040262584\n",
      "164 3498.9456459633275\n",
      "165 3458.755931076613\n",
      "166 3419.313896148891\n",
      "167 3380.517292846975\n",
      "168 3342.4027950874047\n",
      "169 3304.966622924041\n",
      "170 3268.144100500148\n",
      "171 3231.9652141783663\n",
      "172 3196.383580069306\n",
      "173 3161.4010769716224\n",
      "174 3127.0185719287824\n",
      "175 3093.169409170791\n",
      "176 3059.903949033255\n",
      "177 3027.1714786841767\n",
      "178 2994.957665566112\n",
      "179 2963.3091314771245\n",
      "180 2932.147758727702\n",
      "181 2901.481734529699\n",
      "182 2871.307008790843\n",
      "183 2841.4842385651896\n",
      "184 2812.1454082943887\n",
      "185 2783.2485134101744\n",
      "186 2754.8358547987364\n",
      "187 2726.8315564900113\n",
      "188 2699.2578409457424\n",
      "189 2672.1204329355305\n",
      "190 2645.3668659562836\n",
      "191 2619.0410136617384\n",
      "192 2593.0901587524995\n",
      "193 2567.537175488171\n",
      "194 2542.37463458591\n",
      "195 2517.561085655259\n",
      "196 2493.1146686735065\n",
      "197 2469.023728675804\n",
      "198 2445.298477419612\n",
      "199 2421.876714083566\n",
      "200 2398.8297575982424\n",
      "201 2376.0871019542437\n",
      "202 2353.6610856596635\n",
      "203 2331.5165034583506\n",
      "204 2309.668746008515\n",
      "205 2288.1434020176766\n",
      "206 2266.9098391091\n",
      "207 2245.9688833802525\n",
      "208 2225.310390012808\n",
      "209 2204.9440263564466\n",
      "210 2184.836690307238\n",
      "211 2165.0212469775497\n",
      "212 2145.4516876335165\n",
      "213 2126.188091773824\n",
      "214 2107.1895303164356\n",
      "215 2088.44146546287\n",
      "216 2069.95145039572\n",
      "217 2051.7060945872963\n",
      "218 2033.6909134237599\n",
      "219 2015.9069019788483\n",
      "220 1998.3429430913636\n",
      "221 1981.0165082900346\n",
      "222 1963.9037043631788\n",
      "223 1947.0058809109948\n",
      "224 1930.3231407077938\n",
      "225 1913.8481046671022\n",
      "226 1897.5822179492147\n",
      "227 1881.5175419641917\n",
      "228 1865.6537702355527\n",
      "229 1849.9870887421134\n",
      "230 1834.5038226602337\n",
      "231 1819.2213471784808\n",
      "232 1804.121609449014\n",
      "233 1789.2079574965246\n",
      "234 1774.4651640109105\n",
      "235 1759.8995018022947\n",
      "236 1745.502862728812\n",
      "237 1731.2842359587853\n",
      "238 1717.2307128661412\n",
      "239 1703.3415715513845\n",
      "240 1689.6115255698612\n",
      "241 1676.041896721685\n",
      "242 1662.6321849783837\n",
      "243 1649.374195925307\n",
      "244 1636.2707874983362\n",
      "245 1623.313937823548\n",
      "246 1610.5185814830002\n",
      "247 1597.8607687083008\n",
      "248 1585.336353666983\n",
      "249 1572.9579381563367\n",
      "250 1560.7116437900897\n",
      "251 1548.608145491786\n",
      "252 1536.637839883276\n",
      "253 1524.795974747516\n",
      "254 1513.0905741998038\n",
      "255 1501.5067425413313\n",
      "256 1490.0565408027023\n",
      "257 1478.7253357138281\n",
      "258 1467.5192210063974\n",
      "259 1456.434652687418\n",
      "260 1445.4627877090807\n",
      "261 1434.6178762642685\n",
      "262 1423.884287156422\n",
      "263 1413.2614992036881\n",
      "264 1402.7487403176444\n",
      "265 1392.3493604672506\n",
      "266 1382.0542330984947\n",
      "267 1371.8717495127555\n",
      "268 1361.7929024751804\n",
      "269 1351.8182883127902\n",
      "270 1341.9499837380763\n",
      "271 1332.1834548890965\n",
      "272 1322.5102490964155\n",
      "273 1312.9386593644706\n",
      "274 1303.4619852411847\n",
      "275 1294.0886657048131\n",
      "276 1284.804679943374\n",
      "277 1275.6151136534045\n",
      "278 1266.512769298739\n",
      "279 1257.5032081461918\n",
      "280 1248.5817781769686\n",
      "281 1239.7564797120533\n",
      "282 1231.0105863446524\n",
      "283 1222.3577867820281\n",
      "284 1213.7905743056995\n",
      "285 1205.3120727695766\n",
      "286 1196.9122344757288\n",
      "287 1188.5960600824665\n",
      "288 1180.3579696416164\n",
      "289 1172.2057010537694\n",
      "290 1164.1315138249238\n",
      "291 1156.1337437633592\n",
      "292 1148.2049346487292\n",
      "293 1140.3551983056946\n",
      "294 1132.5769894384607\n",
      "295 1124.874236122438\n",
      "296 1117.2418426890201\n",
      "297 1109.684262561223\n",
      "298 1102.194246820441\n",
      "299 1094.7956632832877\n",
      "300 1087.5196572187294\n",
      "301 1080.310198933481\n",
      "302 1073.1677823607104\n",
      "303 1066.0918663987966\n",
      "304 1059.080936410117\n",
      "305 1052.1322233207964\n",
      "306 1045.2487264132228\n",
      "307 1038.4268734072732\n",
      "308 1031.6691578187522\n",
      "309 1024.970254797909\n",
      "310 1018.3310038745915\n",
      "311 1011.7520696148504\n",
      "312 1005.2302980656262\n",
      "313 998.769561244392\n",
      "314 992.3627351133845\n",
      "315 986.0137239466019\n",
      "316 979.7179815826134\n",
      "317 973.4844128494018\n",
      "318 967.3040840363172\n",
      "319 961.1756621153804\n",
      "320 955.0988945757025\n",
      "321 949.072829190072\n",
      "322 943.1123606278271\n",
      "323 937.2090550607668\n",
      "324 931.358461019333\n",
      "325 925.5579187925456\n",
      "326 919.8085695755142\n",
      "327 914.107785887937\n",
      "328 908.4524085995589\n",
      "329 902.8467324439232\n",
      "330 897.2861379429996\n",
      "331 891.7767761952501\n",
      "332 886.3129661602856\n",
      "333 880.8914878387922\n",
      "334 875.5188759052057\n",
      "335 870.1870186584657\n",
      "336 864.9016890123634\n",
      "337 859.6578834135904\n",
      "338 854.4571183662923\n",
      "339 849.3024417368815\n",
      "340 844.1909663978461\n",
      "341 839.1234325060763\n",
      "342 834.0956613658379\n",
      "343 829.109014476004\n",
      "344 824.1626448028015\n",
      "345 819.2587631538746\n",
      "346 814.391706610692\n",
      "347 809.5644989935093\n",
      "348 804.7768021740682\n",
      "349 800.025946467014\n",
      "350 795.2991931691045\n",
      "351 790.6129117387092\n",
      "352 785.9627940856777\n",
      "353 781.3489860444214\n",
      "354 776.7719837364591\n",
      "355 772.2307282333197\n",
      "356 767.7250927875666\n",
      "357 763.2548618469757\n",
      "358 758.8196280016249\n",
      "359 754.4211889006136\n",
      "360 750.0531768357137\n",
      "361 745.7200258463452\n",
      "362 741.4222214651461\n",
      "363 737.1543613590621\n",
      "364 732.920108212461\n",
      "365 728.7194598495977\n",
      "366 724.5493369209911\n",
      "367 720.4121431079974\n",
      "368 716.306351651226\n",
      "369 712.2326048655893\n",
      "370 708.1871790655356\n",
      "371 704.1616398103052\n",
      "372 700.1674997980302\n",
      "373 696.2055931109603\n",
      "374 692.2702950509889\n",
      "375 688.3646028997983\n",
      "376 684.4874232438325\n",
      "377 680.6399797768981\n",
      "378 676.8207903100065\n",
      "379 673.0301434364985\n",
      "380 669.2678836822832\n",
      "381 665.5339209756626\n",
      "382 661.8270371154906\n",
      "383 658.1469154803908\n",
      "384 654.4944147865725\n",
      "385 650.8683114293158\n",
      "386 647.2714693846748\n",
      "387 643.6994736726396\n",
      "388 640.1514509607842\n",
      "389 636.630087026668\n",
      "390 633.1336478008408\n",
      "391 629.6624256549028\n",
      "392 626.216917040558\n",
      "393 622.7971998552797\n",
      "394 619.4014315576776\n",
      "395 616.0296232720912\n",
      "396 612.6858048451886\n",
      "397 609.3668724932737\n",
      "398 606.068276030608\n",
      "399 602.7935616426557\n",
      "400 599.5445179398084\n",
      "401 596.3162438043321\n",
      "402 593.1106290754881\n",
      "403 589.927372409052\n",
      "404 586.7665269236529\n",
      "405 583.6280045588785\n",
      "406 580.5108586483251\n",
      "407 577.4168984507087\n",
      "408 574.3433290282258\n",
      "409 571.2919536733008\n",
      "410 568.2610302472666\n",
      "411 565.2505453381801\n",
      "412 562.2621515401959\n",
      "413 559.2950709421568\n",
      "414 556.347115475535\n",
      "415 553.4195376682073\n",
      "416 550.5116698774074\n",
      "417 547.6244375416378\n",
      "418 544.7561041742651\n",
      "419 541.9073812286108\n",
      "420 539.0797904454408\n",
      "421 536.2895385713895\n",
      "422 533.5184634899985\n",
      "423 530.7662536705919\n",
      "424 528.0327390025764\n",
      "425 525.3171765272526\n",
      "426 522.621082171229\n",
      "427 519.943802953641\n",
      "428 517.2823956497391\n",
      "429 514.6376732260312\n",
      "430 512.0122296019283\n",
      "431 509.4040404975063\n",
      "432 506.81238868064594\n",
      "433 504.2385385422946\n",
      "434 501.6811719152588\n",
      "435 499.1417003309148\n",
      "436 496.61839376377367\n",
      "437 494.11111363315786\n",
      "438 491.6203961022112\n",
      "439 489.1467545216639\n",
      "440 486.6905780194084\n",
      "441 484.2491498358544\n",
      "442 481.82386159208556\n",
      "443 479.4135560320474\n",
      "444 477.02005704992973\n",
      "445 474.64158308465136\n",
      "446 472.2776917060601\n",
      "447 469.9292927997947\n",
      "448 467.59529602410817\n",
      "449 465.2767927428725\n",
      "450 462.9737578297734\n",
      "451 460.68489642971036\n",
      "452 458.4105270953767\n",
      "453 456.1517454292009\n",
      "454 453.90758660499444\n",
      "455 451.6770076983537\n",
      "456 449.46058384187444\n",
      "457 447.25939565512846\n",
      "458 445.07094888361985\n",
      "459 442.89552103311155\n",
      "460 440.7341337825459\n",
      "461 438.5871737297867\n",
      "462 436.45281013670717\n",
      "463 434.3350804270342\n",
      "464 432.23316996691733\n",
      "465 430.14475948384876\n",
      "466 428.0701487668127\n",
      "467 426.0078945468557\n",
      "468 423.9580915250345\n",
      "469 421.92087532223667\n",
      "470 419.8978948409877\n",
      "471 417.88562330908627\n",
      "472 415.8853475775718\n",
      "473 413.8978863139982\n",
      "474 411.922345493206\n",
      "475 409.95897708316875\n",
      "476 408.0078039512787\n",
      "477 406.0684409374788\n",
      "478 404.14130589346\n",
      "479 402.2258916080947\n",
      "480 400.32315464973135\n",
      "481 398.43098296644206\n",
      "482 396.55028399027805\n",
      "483 394.68214725519334\n",
      "484 392.82497333823335\n",
      "485 390.9785937699117\n",
      "486 389.14259594516767\n",
      "487 387.31774332027317\n",
      "488 385.5035518562081\n",
      "489 383.70032934826037\n",
      "490 381.9086319627851\n",
      "491 380.12695168731915\n",
      "492 378.3560226263518\n",
      "493 376.5970160468448\n",
      "494 374.8468275072155\n",
      "495 373.10714770332044\n",
      "496 371.3775897663444\n",
      "497 369.65944963419577\n",
      "498 367.9515878709199\n",
      "499 366.251946240421\n"
     ]
    }
   ],
   "source": [
    "# Numpy Version\n",
    "import numpy as np\n",
    "\n",
    "N, D_in, D_h, D_out = 100, 1000, 100, 1\n",
    "\n",
    "# generate the training data\n",
    "x = np.random.randn(N, D_in)\n",
    "y = np.random.randn(N, D_out)\n",
    "\n",
    "# init the weights\n",
    "w1 = np.random.randn(D_in, D_h)\n",
    "w2 = np.random.randn(D_h, D_out)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "\n",
    "for t in range(500):\n",
    "    # forward inference\n",
    "    h = x.dot(w1)\n",
    "    h_relu = np.maximum(h, 0)\n",
    "    y_pred = h_relu.dot(w2)\n",
    "    # calculate the loss\n",
    "    loss = np.square(y_pred - y).sum()\n",
    "    print(t, loss)\n",
    "    # back-propagation\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.T.dot(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.dot(w2.T)\n",
    "    grad_h = grad_h_relu.copy()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.T.dot(grad_h)\n",
    "    # update weights\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 45237520.0\n",
      "1 98879256.0\n",
      "2 487360192.0\n",
      "3 1159751680.0\n",
      "4 35282896.0\n",
      "5 17066524.0\n",
      "6 11218501.0\n",
      "7 8219145.0\n",
      "8 6383037.0\n",
      "9 5152704.5\n",
      "10 4273749.5\n",
      "11 3620350.25\n",
      "12 3117377.5\n",
      "13 2719061.25\n",
      "14 2398089.25\n",
      "15 2135568.25\n",
      "16 1916720.375\n",
      "17 1732083.5\n",
      "18 1574517.5\n",
      "19 1439053.5\n",
      "20 1320940.0\n",
      "21 1217540.0\n",
      "22 1126268.0\n",
      "23 1045177.75\n",
      "24 973052.6875\n",
      "25 908549.375\n",
      "26 850891.375\n",
      "27 798805.375\n",
      "28 751480.125\n",
      "29 708353.75\n",
      "30 668874.0625\n",
      "31 632678.0625\n",
      "32 599309.75\n",
      "33 568488.5625\n",
      "34 539964.875\n",
      "35 513484.4375\n",
      "36 488958.15625\n",
      "37 466154.5\n",
      "38 444889.6875\n",
      "39 425082.625\n",
      "40 406638.5\n",
      "41 389375.5625\n",
      "42 373156.53125\n",
      "43 357935.625\n",
      "44 343618.96875\n",
      "45 330108.59375\n",
      "46 317370.03125\n",
      "47 305305.875\n",
      "48 293919.0625\n",
      "49 283175.0625\n",
      "50 273020.21875\n",
      "51 263383.0\n",
      "52 254226.5625\n",
      "53 245538.375\n",
      "54 237294.84375\n",
      "55 229470.359375\n",
      "56 222007.234375\n",
      "57 214884.03125\n",
      "58 208096.859375\n",
      "59 201628.265625\n",
      "60 195466.71875\n",
      "61 189572.015625\n",
      "62 183930.78125\n",
      "63 178536.78125\n",
      "64 173373.296875\n",
      "65 168424.8125\n",
      "66 163680.75\n",
      "67 159119.984375\n",
      "68 154747.09375\n",
      "69 150552.5\n",
      "70 146525.234375\n",
      "71 142653.359375\n",
      "72 138930.8125\n",
      "73 135346.28125\n",
      "74 131892.46875\n",
      "75 128566.0625\n",
      "76 125369.2734375\n",
      "77 122292.984375\n",
      "78 119325.2265625\n",
      "79 116465.03125\n",
      "80 113706.359375\n",
      "81 111042.1015625\n",
      "82 108471.4140625\n",
      "83 105987.25\n",
      "84 103587.34375\n",
      "85 101266.046875\n",
      "86 99020.3125\n",
      "87 96847.046875\n",
      "88 94743.859375\n",
      "89 92702.96875\n",
      "90 90724.9765625\n",
      "91 88808.015625\n",
      "92 86950.6328125\n",
      "93 85149.4375\n",
      "94 83407.46875\n",
      "95 81714.9140625\n",
      "96 80072.3515625\n",
      "97 78478.5\n",
      "98 76928.453125\n",
      "99 75424.2421875\n",
      "100 73963.34375\n",
      "101 72545.2890625\n",
      "102 71166.8671875\n",
      "103 69824.359375\n",
      "104 68517.96875\n",
      "105 67248.09375\n",
      "106 66012.0546875\n",
      "107 64808.62109375\n",
      "108 63636.609375\n",
      "109 62496.25390625\n",
      "110 61385.125\n",
      "111 60301.99609375\n",
      "112 59247.6875\n",
      "113 58219.875\n",
      "114 57218.15234375\n",
      "115 56242.30859375\n",
      "116 55290.42578125\n",
      "117 54363.40625\n",
      "118 53459.78515625\n",
      "119 52577.4453125\n",
      "120 51715.62890625\n",
      "121 50874.25\n",
      "122 50053.4453125\n",
      "123 49252.53125\n",
      "124 48469.46484375\n",
      "125 47704.5703125\n",
      "126 46958.265625\n",
      "127 46228.37109375\n",
      "128 45514.64453125\n",
      "129 44818.55859375\n",
      "130 44136.90234375\n",
      "131 43470.1875\n",
      "132 42818.31640625\n",
      "133 42180.71875\n",
      "134 41557.2890625\n",
      "135 40947.5078125\n",
      "136 40350.53125\n",
      "137 39766.9375\n",
      "138 39195.23046875\n",
      "139 38635.26953125\n",
      "140 38086.3984375\n",
      "141 37548.90625\n",
      "142 37023.421875\n",
      "143 36508.4296875\n",
      "144 36003.765625\n",
      "145 35509.01171875\n",
      "146 35024.296875\n",
      "147 34549.8671875\n",
      "148 34085.31640625\n",
      "149 33629.453125\n",
      "150 33182.30078125\n",
      "151 32744.15625\n",
      "152 32314.5546875\n",
      "153 31893.173828125\n",
      "154 31480.26953125\n",
      "155 31075.29296875\n",
      "156 30677.74609375\n",
      "157 30288.107421875\n",
      "158 29906.00390625\n",
      "159 29531.2265625\n",
      "160 29164.11328125\n",
      "161 28803.328125\n",
      "162 28449.26171875\n",
      "163 28101.064453125\n",
      "164 27759.408203125\n",
      "165 27423.84375\n",
      "166 27094.5078125\n",
      "167 26771.3359375\n",
      "168 26453.66796875\n",
      "169 26141.640625\n",
      "170 25835.1171875\n",
      "171 25533.599609375\n",
      "172 25237.49609375\n",
      "173 24946.99609375\n",
      "174 24661.17578125\n",
      "175 24380.30859375\n",
      "176 24103.95703125\n",
      "177 23832.384765625\n",
      "178 23565.56640625\n",
      "179 23302.99609375\n",
      "180 23044.755859375\n",
      "181 22790.6328125\n",
      "182 22540.875\n",
      "183 22294.845703125\n",
      "184 22052.97265625\n",
      "185 21814.9296875\n",
      "186 21580.955078125\n",
      "187 21350.587890625\n",
      "188 21123.841796875\n",
      "189 20900.583984375\n",
      "190 20680.78515625\n",
      "191 20464.53515625\n",
      "192 20251.580078125\n",
      "193 20042.05078125\n",
      "194 19835.748046875\n",
      "195 19632.56640625\n",
      "196 19432.671875\n",
      "197 19235.701171875\n",
      "198 19041.873046875\n",
      "199 18851.03515625\n",
      "200 18662.96875\n",
      "201 18477.796875\n",
      "202 18295.4296875\n",
      "203 18115.62109375\n",
      "204 17938.55078125\n",
      "205 17763.974609375\n",
      "206 17591.994140625\n",
      "207 17422.740234375\n",
      "208 17255.802734375\n",
      "209 17091.296875\n",
      "210 16929.0703125\n",
      "211 16769.232421875\n",
      "212 16611.693359375\n",
      "213 16456.33203125\n",
      "214 16303.4814453125\n",
      "215 16153.21484375\n",
      "216 16004.720703125\n",
      "217 15858.2646484375\n",
      "218 15713.876953125\n",
      "219 15571.5703125\n",
      "220 15431.05859375\n",
      "221 15292.4931640625\n",
      "222 15155.783203125\n",
      "223 15020.9296875\n",
      "224 14887.87890625\n",
      "225 14756.8623046875\n",
      "226 14627.818359375\n",
      "227 14500.419921875\n",
      "228 14374.6669921875\n",
      "229 14250.548828125\n",
      "230 14127.9794921875\n",
      "231 14007.1796875\n",
      "232 13887.9306640625\n",
      "233 13770.265625\n",
      "234 13654.095703125\n",
      "235 13539.4716796875\n",
      "236 13426.341796875\n",
      "237 13314.720703125\n",
      "238 13204.4970703125\n",
      "239 13095.720703125\n",
      "240 12988.5576171875\n",
      "241 12882.55078125\n",
      "242 12777.8046875\n",
      "243 12674.498046875\n",
      "244 12572.37890625\n",
      "245 12471.572265625\n",
      "246 12372.0537109375\n",
      "247 12273.5\n",
      "248 12176.1533203125\n",
      "249 12080.0078125\n",
      "250 11984.986328125\n",
      "251 11891.298828125\n",
      "252 11798.67578125\n",
      "253 11707.3515625\n",
      "254 11617.126953125\n",
      "255 11527.9638671875\n",
      "256 11439.8486328125\n",
      "257 11352.755859375\n",
      "258 11266.7587890625\n",
      "259 11181.8203125\n",
      "260 11097.94140625\n",
      "261 11014.947265625\n",
      "262 10933.0146484375\n",
      "263 10851.986328125\n",
      "264 10771.88671875\n",
      "265 10692.7978515625\n",
      "266 10614.7490234375\n",
      "267 10537.572265625\n",
      "268 10461.216796875\n",
      "269 10385.775390625\n",
      "270 10311.294921875\n",
      "271 10237.576171875\n",
      "272 10164.57421875\n",
      "273 10092.4775390625\n",
      "274 10021.201171875\n",
      "275 9950.68359375\n",
      "276 9880.9990234375\n",
      "277 9812.1787109375\n",
      "278 9744.0302734375\n",
      "279 9676.677734375\n",
      "280 9610.09375\n",
      "281 9544.2236328125\n",
      "282 9479.0517578125\n",
      "283 9414.580078125\n",
      "284 9350.861328125\n",
      "285 9287.8642578125\n",
      "286 9225.44921875\n",
      "287 9163.7412109375\n",
      "288 9102.732421875\n",
      "289 9042.333984375\n",
      "290 8982.6376953125\n",
      "291 8923.564453125\n",
      "292 8865.1162109375\n",
      "293 8807.2197265625\n",
      "294 8749.9638671875\n",
      "295 8693.404296875\n",
      "296 8637.3740234375\n",
      "297 8581.849609375\n",
      "298 8526.9365234375\n",
      "299 8472.640625\n",
      "300 8418.9755859375\n",
      "301 8365.884765625\n",
      "302 8313.3193359375\n",
      "303 8261.318359375\n",
      "304 8209.91796875\n",
      "305 8158.9736328125\n",
      "306 8108.537109375\n",
      "307 8058.62109375\n",
      "308 8009.23828125\n",
      "309 7960.3466796875\n",
      "310 7911.97314453125\n",
      "311 7864.05078125\n",
      "312 7816.60546875\n",
      "313 7769.68798828125\n",
      "314 7723.185546875\n",
      "315 7677.1357421875\n",
      "316 7631.5458984375\n",
      "317 7586.44775390625\n",
      "318 7541.763671875\n",
      "319 7497.57421875\n",
      "320 7453.76220703125\n",
      "321 7410.416015625\n",
      "322 7367.44091796875\n",
      "323 7324.92578125\n",
      "324 7282.84765625\n",
      "325 7241.11328125\n",
      "326 7199.7490234375\n",
      "327 7158.77978515625\n",
      "328 7118.1923828125\n",
      "329 7077.9912109375\n",
      "330 7038.19482421875\n",
      "331 6998.7333984375\n",
      "332 6959.64892578125\n",
      "333 6920.9072265625\n",
      "334 6882.5634765625\n",
      "335 6844.59130859375\n",
      "336 6806.9326171875\n",
      "337 6769.64453125\n",
      "338 6732.720703125\n",
      "339 6696.13330078125\n",
      "340 6659.91552734375\n",
      "341 6624.04296875\n",
      "342 6588.4814453125\n",
      "343 6553.25341796875\n",
      "344 6518.31982421875\n",
      "345 6483.705078125\n",
      "346 6449.44384765625\n",
      "347 6415.43994140625\n",
      "348 6381.7470703125\n",
      "349 6348.34130859375\n",
      "350 6315.287109375\n",
      "351 6282.47265625\n",
      "352 6249.97802734375\n",
      "353 6217.77392578125\n",
      "354 6185.8505859375\n",
      "355 6154.2373046875\n",
      "356 6122.8427734375\n",
      "357 6091.70556640625\n",
      "358 6060.853515625\n",
      "359 6030.271484375\n",
      "360 5999.93896484375\n",
      "361 5969.8837890625\n",
      "362 5940.076171875\n",
      "363 5910.5146484375\n",
      "364 5881.1904296875\n",
      "365 5852.1396484375\n",
      "366 5823.345703125\n",
      "367 5794.7568359375\n",
      "368 5766.408203125\n",
      "369 5738.3125\n",
      "370 5710.4267578125\n",
      "371 5682.75390625\n",
      "372 5655.3544921875\n",
      "373 5628.16455078125\n",
      "374 5601.193359375\n",
      "375 5574.45556640625\n",
      "376 5547.923828125\n",
      "377 5521.611328125\n",
      "378 5495.5078125\n",
      "379 5469.59033203125\n",
      "380 5443.84619140625\n",
      "381 5418.31494140625\n",
      "382 5392.9794921875\n",
      "383 5367.8056640625\n",
      "384 5342.86181640625\n",
      "385 5318.08935546875\n",
      "386 5293.546875\n",
      "387 5269.2109375\n",
      "388 5245.056640625\n",
      "389 5221.119140625\n",
      "390 5197.3291015625\n",
      "391 5173.7314453125\n",
      "392 5150.33203125\n",
      "393 5127.1162109375\n",
      "394 5104.08154296875\n",
      "395 5081.22412109375\n",
      "396 5058.548828125\n",
      "397 5036.076171875\n",
      "398 5013.7451171875\n",
      "399 4991.56982421875\n",
      "400 4969.58203125\n",
      "401 4947.744140625\n",
      "402 4926.08056640625\n",
      "403 4904.61083984375\n",
      "404 4883.296875\n",
      "405 4862.130859375\n",
      "406 4841.1201171875\n",
      "407 4820.2744140625\n",
      "408 4799.58056640625\n",
      "409 4779.04833984375\n",
      "410 4758.67236328125\n",
      "411 4738.44384765625\n",
      "412 4718.4052734375\n",
      "413 4698.46630859375\n",
      "414 4678.67138671875\n",
      "415 4659.02001953125\n",
      "416 4639.4560546875\n",
      "417 4620.04296875\n",
      "418 4600.783203125\n",
      "419 4581.671875\n",
      "420 4562.681640625\n",
      "421 4543.82568359375\n",
      "422 4525.1328125\n",
      "423 4506.5595703125\n",
      "424 4488.14892578125\n",
      "425 4469.830078125\n",
      "426 4451.650390625\n",
      "427 4433.6103515625\n",
      "428 4415.7119140625\n",
      "429 4397.91796875\n",
      "430 4380.2568359375\n",
      "431 4362.71435546875\n",
      "432 4345.28173828125\n",
      "433 4327.982421875\n",
      "434 4310.818359375\n",
      "435 4293.7646484375\n",
      "436 4276.82958984375\n",
      "437 4260.0087890625\n",
      "438 4243.302734375\n",
      "439 4226.6845703125\n",
      "440 4210.1953125\n",
      "441 4193.796875\n",
      "442 4177.5048828125\n",
      "443 4161.3564453125\n",
      "444 4145.31396484375\n",
      "445 4129.38330078125\n",
      "446 4113.564453125\n",
      "447 4097.85400390625\n",
      "448 4082.2333984375\n",
      "449 4066.72802734375\n",
      "450 4051.3349609375\n",
      "451 4036.036376953125\n",
      "452 4020.83056640625\n",
      "453 4005.7529296875\n",
      "454 3990.763671875\n",
      "455 3975.878662109375\n",
      "456 3961.087158203125\n",
      "457 3946.389892578125\n",
      "458 3931.791015625\n",
      "459 3917.28857421875\n",
      "460 3902.89794921875\n",
      "461 3888.5947265625\n",
      "462 3874.37158203125\n",
      "463 3860.247314453125\n",
      "464 3846.212890625\n",
      "465 3832.26513671875\n",
      "466 3818.404296875\n",
      "467 3804.638916015625\n",
      "468 3790.966064453125\n",
      "469 3777.345703125\n",
      "470 3763.8056640625\n",
      "471 3750.35107421875\n",
      "472 3736.98291015625\n",
      "473 3723.700439453125\n",
      "474 3710.4931640625\n",
      "475 3697.381103515625\n",
      "476 3684.340576171875\n",
      "477 3671.38916015625\n",
      "478 3658.51123046875\n",
      "479 3645.724609375\n",
      "480 3633.031005859375\n",
      "481 3620.393798828125\n",
      "482 3607.83740234375\n",
      "483 3595.369140625\n",
      "484 3582.96435546875\n",
      "485 3570.628173828125\n",
      "486 3558.385009765625\n",
      "487 3546.203369140625\n",
      "488 3534.08447265625\n",
      "489 3522.056884765625\n",
      "490 3510.105712890625\n",
      "491 3498.215576171875\n",
      "492 3486.412109375\n",
      "493 3474.66943359375\n",
      "494 3462.99658203125\n",
      "495 3451.36474609375\n",
      "496 3439.730712890625\n",
      "497 3428.1748046875\n",
      "498 3416.705078125\n",
      "499 3405.331787109375\n"
     ]
    }
   ],
   "source": [
    "# Tensor Version\n",
    "import torch\n",
    "\n",
    "# cpu or cuda\n",
    "device = torch.device('cpu')\n",
    "\n",
    "N, D_in, D_h, D_out = 100, 1000, 100, 10\n",
    "\n",
    "# generate the training data\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# init the weights\n",
    "w1 = torch.randn(D_in, D_h, device=device)\n",
    "w2 = torch.randn(D_h, D_out, device=device)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # forward inference\n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    "    # calculate the loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.item())\n",
    "    # back-propagation\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "    # update the weights\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 tensor(52220128., grad_fn=<SumBackward0>)\n",
      "1 tensor(1.1992e+08, grad_fn=<SumBackward0>)\n",
      "2 tensor(4.7306e+08, grad_fn=<SumBackward0>)\n",
      "3 tensor(8.2283e+08, grad_fn=<SumBackward0>)\n",
      "4 tensor(28214168., grad_fn=<SumBackward0>)\n",
      "5 tensor(14514134., grad_fn=<SumBackward0>)\n",
      "6 tensor(9282458., grad_fn=<SumBackward0>)\n",
      "7 tensor(6531954., grad_fn=<SumBackward0>)\n",
      "8 tensor(4863534., grad_fn=<SumBackward0>)\n",
      "9 tensor(3760603.5000, grad_fn=<SumBackward0>)\n",
      "10 tensor(2989122., grad_fn=<SumBackward0>)\n",
      "11 tensor(2428747.2500, grad_fn=<SumBackward0>)\n",
      "12 tensor(2009219.5000, grad_fn=<SumBackward0>)\n",
      "13 tensor(1687784.5000, grad_fn=<SumBackward0>)\n",
      "14 tensor(1436648., grad_fn=<SumBackward0>)\n",
      "15 tensor(1237514.3750, grad_fn=<SumBackward0>)\n",
      "16 tensor(1077442.8750, grad_fn=<SumBackward0>)\n",
      "17 tensor(947199.6250, grad_fn=<SumBackward0>)\n",
      "18 tensor(840193.3750, grad_fn=<SumBackward0>)\n",
      "19 tensor(751451.7500, grad_fn=<SumBackward0>)\n",
      "20 tensor(677170.1250, grad_fn=<SumBackward0>)\n",
      "21 tensor(614356.6875, grad_fn=<SumBackward0>)\n",
      "22 tensor(560861.7500, grad_fn=<SumBackward0>)\n",
      "23 tensor(514952.5625, grad_fn=<SumBackward0>)\n",
      "24 tensor(475271.8125, grad_fn=<SumBackward0>)\n",
      "25 tensor(440775.0625, grad_fn=<SumBackward0>)\n",
      "26 tensor(410543.6875, grad_fn=<SumBackward0>)\n",
      "27 tensor(383829.6250, grad_fn=<SumBackward0>)\n",
      "28 tensor(359978.5938, grad_fn=<SumBackward0>)\n",
      "29 tensor(338645.6250, grad_fn=<SumBackward0>)\n",
      "30 tensor(319441.1250, grad_fn=<SumBackward0>)\n",
      "31 tensor(302073.3125, grad_fn=<SumBackward0>)\n",
      "32 tensor(286335.1875, grad_fn=<SumBackward0>)\n",
      "33 tensor(271978.5000, grad_fn=<SumBackward0>)\n",
      "34 tensor(258818.3750, grad_fn=<SumBackward0>)\n",
      "35 tensor(246695.8594, grad_fn=<SumBackward0>)\n",
      "36 tensor(235498.0312, grad_fn=<SumBackward0>)\n",
      "37 tensor(225122.5938, grad_fn=<SumBackward0>)\n",
      "38 tensor(215480., grad_fn=<SumBackward0>)\n",
      "39 tensor(206490.2969, grad_fn=<SumBackward0>)\n",
      "40 tensor(198089.0938, grad_fn=<SumBackward0>)\n",
      "41 tensor(190216.2812, grad_fn=<SumBackward0>)\n",
      "42 tensor(182833.7969, grad_fn=<SumBackward0>)\n",
      "43 tensor(175884.4219, grad_fn=<SumBackward0>)\n",
      "44 tensor(169344.5625, grad_fn=<SumBackward0>)\n",
      "45 tensor(163170.3438, grad_fn=<SumBackward0>)\n",
      "46 tensor(157352.5938, grad_fn=<SumBackward0>)\n",
      "47 tensor(151845.6562, grad_fn=<SumBackward0>)\n",
      "48 tensor(146627.8750, grad_fn=<SumBackward0>)\n",
      "49 tensor(141681.0156, grad_fn=<SumBackward0>)\n",
      "50 tensor(136981.0312, grad_fn=<SumBackward0>)\n",
      "51 tensor(132512.1875, grad_fn=<SumBackward0>)\n",
      "52 tensor(128257.4219, grad_fn=<SumBackward0>)\n",
      "53 tensor(124204.4453, grad_fn=<SumBackward0>)\n",
      "54 tensor(120318.0625, grad_fn=<SumBackward0>)\n",
      "55 tensor(116618.5938, grad_fn=<SumBackward0>)\n",
      "56 tensor(113086.6875, grad_fn=<SumBackward0>)\n",
      "57 tensor(109711.8594, grad_fn=<SumBackward0>)\n",
      "58 tensor(106482.6562, grad_fn=<SumBackward0>)\n",
      "59 tensor(103392.6328, grad_fn=<SumBackward0>)\n",
      "60 tensor(100432.9844, grad_fn=<SumBackward0>)\n",
      "61 tensor(97595.8672, grad_fn=<SumBackward0>)\n",
      "62 tensor(94877.1797, grad_fn=<SumBackward0>)\n",
      "63 tensor(92269.1094, grad_fn=<SumBackward0>)\n",
      "64 tensor(89764.8906, grad_fn=<SumBackward0>)\n",
      "65 tensor(87359.2656, grad_fn=<SumBackward0>)\n",
      "66 tensor(85046.7969, grad_fn=<SumBackward0>)\n",
      "67 tensor(82825.0469, grad_fn=<SumBackward0>)\n",
      "68 tensor(80688.0312, grad_fn=<SumBackward0>)\n",
      "69 tensor(78633.4844, grad_fn=<SumBackward0>)\n",
      "70 tensor(76655.9609, grad_fn=<SumBackward0>)\n",
      "71 tensor(74749.3984, grad_fn=<SumBackward0>)\n",
      "72 tensor(72910.8438, grad_fn=<SumBackward0>)\n",
      "73 tensor(71136.7500, grad_fn=<SumBackward0>)\n",
      "74 tensor(69424.3750, grad_fn=<SumBackward0>)\n",
      "75 tensor(67771.6094, grad_fn=<SumBackward0>)\n",
      "76 tensor(66174.7266, grad_fn=<SumBackward0>)\n",
      "77 tensor(64630.4766, grad_fn=<SumBackward0>)\n",
      "78 tensor(63139.2344, grad_fn=<SumBackward0>)\n",
      "79 tensor(61697.7539, grad_fn=<SumBackward0>)\n",
      "80 tensor(60303.8750, grad_fn=<SumBackward0>)\n",
      "81 tensor(58954.8477, grad_fn=<SumBackward0>)\n",
      "82 tensor(57648.6406, grad_fn=<SumBackward0>)\n",
      "83 tensor(56382.6133, grad_fn=<SumBackward0>)\n",
      "84 tensor(55155.8516, grad_fn=<SumBackward0>)\n",
      "85 tensor(53967.2969, grad_fn=<SumBackward0>)\n",
      "86 tensor(52813.9297, grad_fn=<SumBackward0>)\n",
      "87 tensor(51695.3359, grad_fn=<SumBackward0>)\n",
      "88 tensor(50610., grad_fn=<SumBackward0>)\n",
      "89 tensor(49555.6641, grad_fn=<SumBackward0>)\n",
      "90 tensor(48532.5547, grad_fn=<SumBackward0>)\n",
      "91 tensor(47538.2656, grad_fn=<SumBackward0>)\n",
      "92 tensor(46571.8750, grad_fn=<SumBackward0>)\n",
      "93 tensor(45633.1211, grad_fn=<SumBackward0>)\n",
      "94 tensor(44719.8398, grad_fn=<SumBackward0>)\n",
      "95 tensor(43831.9258, grad_fn=<SumBackward0>)\n",
      "96 tensor(42968.2148, grad_fn=<SumBackward0>)\n",
      "97 tensor(42127.4531, grad_fn=<SumBackward0>)\n",
      "98 tensor(41309.2227, grad_fn=<SumBackward0>)\n",
      "99 tensor(40512.5742, grad_fn=<SumBackward0>)\n",
      "100 tensor(39738.4766, grad_fn=<SumBackward0>)\n",
      "101 tensor(38986.4688, grad_fn=<SumBackward0>)\n",
      "102 tensor(38254.2109, grad_fn=<SumBackward0>)\n",
      "103 tensor(37539.9492, grad_fn=<SumBackward0>)\n",
      "104 tensor(36844.0156, grad_fn=<SumBackward0>)\n",
      "105 tensor(36166.4648, grad_fn=<SumBackward0>)\n",
      "106 tensor(35505.5117, grad_fn=<SumBackward0>)\n",
      "107 tensor(34860.2969, grad_fn=<SumBackward0>)\n",
      "108 tensor(34230.1250, grad_fn=<SumBackward0>)\n",
      "109 tensor(33615.5312, grad_fn=<SumBackward0>)\n",
      "110 tensor(33016.1953, grad_fn=<SumBackward0>)\n",
      "111 tensor(32431.8535, grad_fn=<SumBackward0>)\n",
      "112 tensor(31861.8809, grad_fn=<SumBackward0>)\n",
      "113 tensor(31305.4375, grad_fn=<SumBackward0>)\n",
      "114 tensor(30761.9824, grad_fn=<SumBackward0>)\n",
      "115 tensor(30231.2812, grad_fn=<SumBackward0>)\n",
      "116 tensor(29712.8301, grad_fn=<SumBackward0>)\n",
      "117 tensor(29206.0840, grad_fn=<SumBackward0>)\n",
      "118 tensor(28711.1953, grad_fn=<SumBackward0>)\n",
      "119 tensor(28227.6680, grad_fn=<SumBackward0>)\n",
      "120 tensor(27755.2891, grad_fn=<SumBackward0>)\n",
      "121 tensor(27293.5996, grad_fn=<SumBackward0>)\n",
      "122 tensor(26842.3672, grad_fn=<SumBackward0>)\n",
      "123 tensor(26401.0840, grad_fn=<SumBackward0>)\n",
      "124 tensor(25969.4629, grad_fn=<SumBackward0>)\n",
      "125 tensor(25547.1719, grad_fn=<SumBackward0>)\n",
      "126 tensor(25133.9473, grad_fn=<SumBackward0>)\n",
      "127 tensor(24730.0664, grad_fn=<SumBackward0>)\n",
      "128 tensor(24334.6172, grad_fn=<SumBackward0>)\n",
      "129 tensor(23947.7910, grad_fn=<SumBackward0>)\n",
      "130 tensor(23569.1328, grad_fn=<SumBackward0>)\n",
      "131 tensor(23198.9727, grad_fn=<SumBackward0>)\n",
      "132 tensor(22836.2285, grad_fn=<SumBackward0>)\n",
      "133 tensor(22481.1445, grad_fn=<SumBackward0>)\n",
      "134 tensor(22133.3965, grad_fn=<SumBackward0>)\n",
      "135 tensor(21792.8965, grad_fn=<SumBackward0>)\n",
      "136 tensor(21459.6074, grad_fn=<SumBackward0>)\n",
      "137 tensor(21132.9844, grad_fn=<SumBackward0>)\n",
      "138 tensor(20813.0703, grad_fn=<SumBackward0>)\n",
      "139 tensor(20497.2070, grad_fn=<SumBackward0>)\n",
      "140 tensor(20187.6504, grad_fn=<SumBackward0>)\n",
      "141 tensor(19884.3281, grad_fn=<SumBackward0>)\n",
      "142 tensor(19586.8242, grad_fn=<SumBackward0>)\n",
      "143 tensor(19292.0215, grad_fn=<SumBackward0>)\n",
      "144 tensor(19003.3281, grad_fn=<SumBackward0>)\n",
      "145 tensor(18720.3945, grad_fn=<SumBackward0>)\n",
      "146 tensor(18442.6797, grad_fn=<SumBackward0>)\n",
      "147 tensor(18170.4922, grad_fn=<SumBackward0>)\n",
      "148 tensor(17903.5645, grad_fn=<SumBackward0>)\n",
      "149 tensor(17641.8027, grad_fn=<SumBackward0>)\n",
      "150 tensor(17385.0977, grad_fn=<SumBackward0>)\n",
      "151 tensor(17133.0625, grad_fn=<SumBackward0>)\n",
      "152 tensor(16885.8848, grad_fn=<SumBackward0>)\n",
      "153 tensor(16642.8398, grad_fn=<SumBackward0>)\n",
      "154 tensor(16404.3203, grad_fn=<SumBackward0>)\n",
      "155 tensor(16170.2295, grad_fn=<SumBackward0>)\n",
      "156 tensor(15941.0137, grad_fn=<SumBackward0>)\n",
      "157 tensor(15716.5371, grad_fn=<SumBackward0>)\n",
      "158 tensor(15497.5312, grad_fn=<SumBackward0>)\n",
      "159 tensor(15282.8125, grad_fn=<SumBackward0>)\n",
      "160 tensor(15072.0674, grad_fn=<SumBackward0>)\n",
      "161 tensor(14864.9395, grad_fn=<SumBackward0>)\n",
      "162 tensor(14661.7041, grad_fn=<SumBackward0>)\n",
      "163 tensor(14462.1016, grad_fn=<SumBackward0>)\n",
      "164 tensor(14266.0625, grad_fn=<SumBackward0>)\n",
      "165 tensor(14073.4668, grad_fn=<SumBackward0>)\n",
      "166 tensor(13884.3936, grad_fn=<SumBackward0>)\n",
      "167 tensor(13698.4268, grad_fn=<SumBackward0>)\n",
      "168 tensor(13515.6553, grad_fn=<SumBackward0>)\n",
      "169 tensor(13335.5625, grad_fn=<SumBackward0>)\n",
      "170 tensor(13158.7852, grad_fn=<SumBackward0>)\n",
      "171 tensor(12985.0225, grad_fn=<SumBackward0>)\n",
      "172 tensor(12814.1152, grad_fn=<SumBackward0>)\n",
      "173 tensor(12646.0469, grad_fn=<SumBackward0>)\n",
      "174 tensor(12481.1270, grad_fn=<SumBackward0>)\n",
      "175 tensor(12319.1582, grad_fn=<SumBackward0>)\n",
      "176 tensor(12159.8652, grad_fn=<SumBackward0>)\n",
      "177 tensor(12003.4297, grad_fn=<SumBackward0>)\n",
      "178 tensor(11849.4834, grad_fn=<SumBackward0>)\n",
      "179 tensor(11698.1387, grad_fn=<SumBackward0>)\n",
      "180 tensor(11549.2559, grad_fn=<SumBackward0>)\n",
      "181 tensor(11402.8008, grad_fn=<SumBackward0>)\n",
      "182 tensor(11258.9473, grad_fn=<SumBackward0>)\n",
      "183 tensor(11117.3926, grad_fn=<SumBackward0>)\n",
      "184 tensor(10978.0664, grad_fn=<SumBackward0>)\n",
      "185 tensor(10840.9824, grad_fn=<SumBackward0>)\n",
      "186 tensor(10706.2363, grad_fn=<SumBackward0>)\n",
      "187 tensor(10573.5615, grad_fn=<SumBackward0>)\n",
      "188 tensor(10443.0400, grad_fn=<SumBackward0>)\n",
      "189 tensor(10314.6758, grad_fn=<SumBackward0>)\n",
      "190 tensor(10188.4111, grad_fn=<SumBackward0>)\n",
      "191 tensor(10064.1562, grad_fn=<SumBackward0>)\n",
      "192 tensor(9941.7920, grad_fn=<SumBackward0>)\n",
      "193 tensor(9821.3936, grad_fn=<SumBackward0>)\n",
      "194 tensor(9702.8701, grad_fn=<SumBackward0>)\n",
      "195 tensor(9586.1875, grad_fn=<SumBackward0>)\n",
      "196 tensor(9471.3037, grad_fn=<SumBackward0>)\n",
      "197 tensor(9358.1104, grad_fn=<SumBackward0>)\n",
      "198 tensor(9246.7129, grad_fn=<SumBackward0>)\n",
      "199 tensor(9137.0547, grad_fn=<SumBackward0>)\n",
      "200 tensor(9029.1660, grad_fn=<SumBackward0>)\n",
      "201 tensor(8922.8232, grad_fn=<SumBackward0>)\n",
      "202 tensor(8818.0146, grad_fn=<SumBackward0>)\n",
      "203 tensor(8714.8135, grad_fn=<SumBackward0>)\n",
      "204 tensor(8613.1475, grad_fn=<SumBackward0>)\n",
      "205 tensor(8513.0166, grad_fn=<SumBackward0>)\n",
      "206 tensor(8414.3779, grad_fn=<SumBackward0>)\n",
      "207 tensor(8317.2041, grad_fn=<SumBackward0>)\n",
      "208 tensor(8221.4199, grad_fn=<SumBackward0>)\n",
      "209 tensor(8127.0420, grad_fn=<SumBackward0>)\n",
      "210 tensor(8034.0371, grad_fn=<SumBackward0>)\n",
      "211 tensor(7942.4600, grad_fn=<SumBackward0>)\n",
      "212 tensor(7852.1484, grad_fn=<SumBackward0>)\n",
      "213 tensor(7763.0835, grad_fn=<SumBackward0>)\n",
      "214 tensor(7675.2832, grad_fn=<SumBackward0>)\n",
      "215 tensor(7588.7871, grad_fn=<SumBackward0>)\n",
      "216 tensor(7503.4863, grad_fn=<SumBackward0>)\n",
      "217 tensor(7419.5898, grad_fn=<SumBackward0>)\n",
      "218 tensor(7337.0693, grad_fn=<SumBackward0>)\n",
      "219 tensor(7255.6904, grad_fn=<SumBackward0>)\n",
      "220 tensor(7175.5830, grad_fn=<SumBackward0>)\n",
      "221 tensor(7096.7295, grad_fn=<SumBackward0>)\n",
      "222 tensor(7019.0293, grad_fn=<SumBackward0>)\n",
      "223 tensor(6942.4595, grad_fn=<SumBackward0>)\n",
      "224 tensor(6866.9448, grad_fn=<SumBackward0>)\n",
      "225 tensor(6792.4932, grad_fn=<SumBackward0>)\n",
      "226 tensor(6719.1133, grad_fn=<SumBackward0>)\n",
      "227 tensor(6646.7100, grad_fn=<SumBackward0>)\n",
      "228 tensor(6575.3413, grad_fn=<SumBackward0>)\n",
      "229 tensor(6504.9238, grad_fn=<SumBackward0>)\n",
      "230 tensor(6435.4604, grad_fn=<SumBackward0>)\n",
      "231 tensor(6366.8984, grad_fn=<SumBackward0>)\n",
      "232 tensor(6299.2754, grad_fn=<SumBackward0>)\n",
      "233 tensor(6232.6021, grad_fn=<SumBackward0>)\n",
      "234 tensor(6166.8613, grad_fn=<SumBackward0>)\n",
      "235 tensor(6101.9819, grad_fn=<SumBackward0>)\n",
      "236 tensor(6037.9736, grad_fn=<SumBackward0>)\n",
      "237 tensor(5974.8389, grad_fn=<SumBackward0>)\n",
      "238 tensor(5912.5576, grad_fn=<SumBackward0>)\n",
      "239 tensor(5851.0898, grad_fn=<SumBackward0>)\n",
      "240 tensor(5790.4473, grad_fn=<SumBackward0>)\n",
      "241 tensor(5730.6572, grad_fn=<SumBackward0>)\n",
      "242 tensor(5671.7285, grad_fn=<SumBackward0>)\n",
      "243 tensor(5613.5518, grad_fn=<SumBackward0>)\n",
      "244 tensor(5556.1167, grad_fn=<SumBackward0>)\n",
      "245 tensor(5499.4619, grad_fn=<SumBackward0>)\n",
      "246 tensor(5443.5220, grad_fn=<SumBackward0>)\n",
      "247 tensor(5388.2817, grad_fn=<SumBackward0>)\n",
      "248 tensor(5333.7896, grad_fn=<SumBackward0>)\n",
      "249 tensor(5280.0981, grad_fn=<SumBackward0>)\n",
      "250 tensor(5227.1294, grad_fn=<SumBackward0>)\n",
      "251 tensor(5174.8027, grad_fn=<SumBackward0>)\n",
      "252 tensor(5123.1953, grad_fn=<SumBackward0>)\n",
      "253 tensor(5072.2061, grad_fn=<SumBackward0>)\n",
      "254 tensor(5021.9385, grad_fn=<SumBackward0>)\n",
      "255 tensor(4972.3574, grad_fn=<SumBackward0>)\n",
      "256 tensor(4923.3921, grad_fn=<SumBackward0>)\n",
      "257 tensor(4875.0176, grad_fn=<SumBackward0>)\n",
      "258 tensor(4827.2812, grad_fn=<SumBackward0>)\n",
      "259 tensor(4780.1201, grad_fn=<SumBackward0>)\n",
      "260 tensor(4733.5234, grad_fn=<SumBackward0>)\n",
      "261 tensor(4687.5117, grad_fn=<SumBackward0>)\n",
      "262 tensor(4642.0918, grad_fn=<SumBackward0>)\n",
      "263 tensor(4597.2026, grad_fn=<SumBackward0>)\n",
      "264 tensor(4552.8589, grad_fn=<SumBackward0>)\n",
      "265 tensor(4509.0581, grad_fn=<SumBackward0>)\n",
      "266 tensor(4465.7847, grad_fn=<SumBackward0>)\n",
      "267 tensor(4423.0215, grad_fn=<SumBackward0>)\n",
      "268 tensor(4380.7681, grad_fn=<SumBackward0>)\n",
      "269 tensor(4339.0322, grad_fn=<SumBackward0>)\n",
      "270 tensor(4297.8096, grad_fn=<SumBackward0>)\n",
      "271 tensor(4257.0713, grad_fn=<SumBackward0>)\n",
      "272 tensor(4216.7998, grad_fn=<SumBackward0>)\n",
      "273 tensor(4177.0249, grad_fn=<SumBackward0>)\n",
      "274 tensor(4137.7314, grad_fn=<SumBackward0>)\n",
      "275 tensor(4098.9053, grad_fn=<SumBackward0>)\n",
      "276 tensor(4060.5234, grad_fn=<SumBackward0>)\n",
      "277 tensor(4022.5874, grad_fn=<SumBackward0>)\n",
      "278 tensor(3985.1089, grad_fn=<SumBackward0>)\n",
      "279 tensor(3948.0879, grad_fn=<SumBackward0>)\n",
      "280 tensor(3911.5005, grad_fn=<SumBackward0>)\n",
      "281 tensor(3875.3193, grad_fn=<SumBackward0>)\n",
      "282 tensor(3839.5637, grad_fn=<SumBackward0>)\n",
      "283 tensor(3804.2383, grad_fn=<SumBackward0>)\n",
      "284 tensor(3769.3008, grad_fn=<SumBackward0>)\n",
      "285 tensor(3734.7690, grad_fn=<SumBackward0>)\n",
      "286 tensor(3700.6284, grad_fn=<SumBackward0>)\n",
      "287 tensor(3666.8823, grad_fn=<SumBackward0>)\n",
      "288 tensor(3633.6023, grad_fn=<SumBackward0>)\n",
      "289 tensor(3600.7466, grad_fn=<SumBackward0>)\n",
      "290 tensor(3568.2544, grad_fn=<SumBackward0>)\n",
      "291 tensor(3536.1333, grad_fn=<SumBackward0>)\n",
      "292 tensor(3504.3660, grad_fn=<SumBackward0>)\n",
      "293 tensor(3472.9575, grad_fn=<SumBackward0>)\n",
      "294 tensor(3441.8945, grad_fn=<SumBackward0>)\n",
      "295 tensor(3411.2095, grad_fn=<SumBackward0>)\n",
      "296 tensor(3380.8694, grad_fn=<SumBackward0>)\n",
      "297 tensor(3350.8398, grad_fn=<SumBackward0>)\n",
      "298 tensor(3321.1401, grad_fn=<SumBackward0>)\n",
      "299 tensor(3291.7705, grad_fn=<SumBackward0>)\n",
      "300 tensor(3262.7263, grad_fn=<SumBackward0>)\n",
      "301 tensor(3234.0117, grad_fn=<SumBackward0>)\n",
      "302 tensor(3205.6157, grad_fn=<SumBackward0>)\n",
      "303 tensor(3177.5166, grad_fn=<SumBackward0>)\n",
      "304 tensor(3149.7319, grad_fn=<SumBackward0>)\n",
      "305 tensor(3122.2432, grad_fn=<SumBackward0>)\n",
      "306 tensor(3095.0479, grad_fn=<SumBackward0>)\n",
      "307 tensor(3068.1626, grad_fn=<SumBackward0>)\n",
      "308 tensor(3041.5625, grad_fn=<SumBackward0>)\n",
      "309 tensor(3015.2954, grad_fn=<SumBackward0>)\n",
      "310 tensor(2989.3345, grad_fn=<SumBackward0>)\n",
      "311 tensor(2963.6089, grad_fn=<SumBackward0>)\n",
      "312 tensor(2938.1536, grad_fn=<SumBackward0>)\n",
      "313 tensor(2912.9863, grad_fn=<SumBackward0>)\n",
      "314 tensor(2888.0828, grad_fn=<SumBackward0>)\n",
      "315 tensor(2863.4480, grad_fn=<SumBackward0>)\n",
      "316 tensor(2839.0908, grad_fn=<SumBackward0>)\n",
      "317 tensor(2814.9705, grad_fn=<SumBackward0>)\n",
      "318 tensor(2791.1060, grad_fn=<SumBackward0>)\n",
      "319 tensor(2767.5005, grad_fn=<SumBackward0>)\n",
      "320 tensor(2744.1367, grad_fn=<SumBackward0>)\n",
      "321 tensor(2721.0967, grad_fn=<SumBackward0>)\n",
      "322 tensor(2698.3315, grad_fn=<SumBackward0>)\n",
      "323 tensor(2675.8069, grad_fn=<SumBackward0>)\n",
      "324 tensor(2653.5132, grad_fn=<SumBackward0>)\n",
      "325 tensor(2631.4592, grad_fn=<SumBackward0>)\n",
      "326 tensor(2609.6335, grad_fn=<SumBackward0>)\n",
      "327 tensor(2588.0371, grad_fn=<SumBackward0>)\n",
      "328 tensor(2566.6899, grad_fn=<SumBackward0>)\n",
      "329 tensor(2545.5381, grad_fn=<SumBackward0>)\n",
      "330 tensor(2524.6006, grad_fn=<SumBackward0>)\n",
      "331 tensor(2503.8843, grad_fn=<SumBackward0>)\n",
      "332 tensor(2483.3887, grad_fn=<SumBackward0>)\n",
      "333 tensor(2463.1082, grad_fn=<SumBackward0>)\n",
      "334 tensor(2443.0212, grad_fn=<SumBackward0>)\n",
      "335 tensor(2423.1416, grad_fn=<SumBackward0>)\n",
      "336 tensor(2403.4790, grad_fn=<SumBackward0>)\n",
      "337 tensor(2384.0425, grad_fn=<SumBackward0>)\n",
      "338 tensor(2364.8474, grad_fn=<SumBackward0>)\n",
      "339 tensor(2345.8423, grad_fn=<SumBackward0>)\n",
      "340 tensor(2327.0410, grad_fn=<SumBackward0>)\n",
      "341 tensor(2308.4253, grad_fn=<SumBackward0>)\n",
      "342 tensor(2290.0005, grad_fn=<SumBackward0>)\n",
      "343 tensor(2271.7549, grad_fn=<SumBackward0>)\n",
      "344 tensor(2253.6956, grad_fn=<SumBackward0>)\n",
      "345 tensor(2235.8218, grad_fn=<SumBackward0>)\n",
      "346 tensor(2218.1345, grad_fn=<SumBackward0>)\n",
      "347 tensor(2200.6235, grad_fn=<SumBackward0>)\n",
      "348 tensor(2183.2695, grad_fn=<SumBackward0>)\n",
      "349 tensor(2166.0930, grad_fn=<SumBackward0>)\n",
      "350 tensor(2149.0842, grad_fn=<SumBackward0>)\n",
      "351 tensor(2132.2478, grad_fn=<SumBackward0>)\n",
      "352 tensor(2115.5854, grad_fn=<SumBackward0>)\n",
      "353 tensor(2099.0803, grad_fn=<SumBackward0>)\n",
      "354 tensor(2082.7385, grad_fn=<SumBackward0>)\n",
      "355 tensor(2066.5537, grad_fn=<SumBackward0>)\n",
      "356 tensor(2050.5234, grad_fn=<SumBackward0>)\n",
      "357 tensor(2034.6565, grad_fn=<SumBackward0>)\n",
      "358 tensor(2018.9498, grad_fn=<SumBackward0>)\n",
      "359 tensor(2003.3843, grad_fn=<SumBackward0>)\n",
      "360 tensor(1987.9772, grad_fn=<SumBackward0>)\n",
      "361 tensor(1972.7173, grad_fn=<SumBackward0>)\n",
      "362 tensor(1957.6005, grad_fn=<SumBackward0>)\n",
      "363 tensor(1942.6223, grad_fn=<SumBackward0>)\n",
      "364 tensor(1927.8060, grad_fn=<SumBackward0>)\n",
      "365 tensor(1913.1185, grad_fn=<SumBackward0>)\n",
      "366 tensor(1898.5640, grad_fn=<SumBackward0>)\n",
      "367 tensor(1884.1252, grad_fn=<SumBackward0>)\n",
      "368 tensor(1869.8259, grad_fn=<SumBackward0>)\n",
      "369 tensor(1855.6575, grad_fn=<SumBackward0>)\n",
      "370 tensor(1841.6290, grad_fn=<SumBackward0>)\n",
      "371 tensor(1827.7317, grad_fn=<SumBackward0>)\n",
      "372 tensor(1813.9688, grad_fn=<SumBackward0>)\n",
      "373 tensor(1800.3412, grad_fn=<SumBackward0>)\n",
      "374 tensor(1786.8351, grad_fn=<SumBackward0>)\n",
      "375 tensor(1773.4623, grad_fn=<SumBackward0>)\n",
      "376 tensor(1760.2131, grad_fn=<SumBackward0>)\n",
      "377 tensor(1747.0873, grad_fn=<SumBackward0>)\n",
      "378 tensor(1734.0822, grad_fn=<SumBackward0>)\n",
      "379 tensor(1721.1980, grad_fn=<SumBackward0>)\n",
      "380 tensor(1708.4370, grad_fn=<SumBackward0>)\n",
      "381 tensor(1695.7919, grad_fn=<SumBackward0>)\n",
      "382 tensor(1683.2689, grad_fn=<SumBackward0>)\n",
      "383 tensor(1670.8584, grad_fn=<SumBackward0>)\n",
      "384 tensor(1658.5676, grad_fn=<SumBackward0>)\n",
      "385 tensor(1646.3876, grad_fn=<SumBackward0>)\n",
      "386 tensor(1634.3198, grad_fn=<SumBackward0>)\n",
      "387 tensor(1622.3589, grad_fn=<SumBackward0>)\n",
      "388 tensor(1610.5089, grad_fn=<SumBackward0>)\n",
      "389 tensor(1598.7694, grad_fn=<SumBackward0>)\n",
      "390 tensor(1587.1315, grad_fn=<SumBackward0>)\n",
      "391 tensor(1575.6044, grad_fn=<SumBackward0>)\n",
      "392 tensor(1564.1821, grad_fn=<SumBackward0>)\n",
      "393 tensor(1552.8647, grad_fn=<SumBackward0>)\n",
      "394 tensor(1541.6460, grad_fn=<SumBackward0>)\n",
      "395 tensor(1530.5293, grad_fn=<SumBackward0>)\n",
      "396 tensor(1519.5122, grad_fn=<SumBackward0>)\n",
      "397 tensor(1508.6006, grad_fn=<SumBackward0>)\n",
      "398 tensor(1497.7856, grad_fn=<SumBackward0>)\n",
      "399 tensor(1487.0662, grad_fn=<SumBackward0>)\n",
      "400 tensor(1476.4425, grad_fn=<SumBackward0>)\n",
      "401 tensor(1465.9117, grad_fn=<SumBackward0>)\n",
      "402 tensor(1455.4773, grad_fn=<SumBackward0>)\n",
      "403 tensor(1445.1364, grad_fn=<SumBackward0>)\n",
      "404 tensor(1434.8846, grad_fn=<SumBackward0>)\n",
      "405 tensor(1424.7234, grad_fn=<SumBackward0>)\n",
      "406 tensor(1414.6510, grad_fn=<SumBackward0>)\n",
      "407 tensor(1404.6685, grad_fn=<SumBackward0>)\n",
      "408 tensor(1394.7769, grad_fn=<SumBackward0>)\n",
      "409 tensor(1384.9784, grad_fn=<SumBackward0>)\n",
      "410 tensor(1375.2548, grad_fn=<SumBackward0>)\n",
      "411 tensor(1365.6174, grad_fn=<SumBackward0>)\n",
      "412 tensor(1356.0652, grad_fn=<SumBackward0>)\n",
      "413 tensor(1346.5958, grad_fn=<SumBackward0>)\n",
      "414 tensor(1337.2059, grad_fn=<SumBackward0>)\n",
      "415 tensor(1327.9045, grad_fn=<SumBackward0>)\n",
      "416 tensor(1318.6807, grad_fn=<SumBackward0>)\n",
      "417 tensor(1309.5392, grad_fn=<SumBackward0>)\n",
      "418 tensor(1300.4785, grad_fn=<SumBackward0>)\n",
      "419 tensor(1291.4937, grad_fn=<SumBackward0>)\n",
      "420 tensor(1282.5825, grad_fn=<SumBackward0>)\n",
      "421 tensor(1273.7527, grad_fn=<SumBackward0>)\n",
      "422 tensor(1264.9987, grad_fn=<SumBackward0>)\n",
      "423 tensor(1256.3204, grad_fn=<SumBackward0>)\n",
      "424 tensor(1247.7146, grad_fn=<SumBackward0>)\n",
      "425 tensor(1239.1813, grad_fn=<SumBackward0>)\n",
      "426 tensor(1230.7222, grad_fn=<SumBackward0>)\n",
      "427 tensor(1222.3376, grad_fn=<SumBackward0>)\n",
      "428 tensor(1214.0250, grad_fn=<SumBackward0>)\n",
      "429 tensor(1205.7815, grad_fn=<SumBackward0>)\n",
      "430 tensor(1197.6064, grad_fn=<SumBackward0>)\n",
      "431 tensor(1189.5017, grad_fn=<SumBackward0>)\n",
      "432 tensor(1181.4635, grad_fn=<SumBackward0>)\n",
      "433 tensor(1173.4968, grad_fn=<SumBackward0>)\n",
      "434 tensor(1165.5944, grad_fn=<SumBackward0>)\n",
      "435 tensor(1157.7916, grad_fn=<SumBackward0>)\n",
      "436 tensor(1150.0813, grad_fn=<SumBackward0>)\n",
      "437 tensor(1142.4327, grad_fn=<SumBackward0>)\n",
      "438 tensor(1134.8479, grad_fn=<SumBackward0>)\n",
      "439 tensor(1127.3298, grad_fn=<SumBackward0>)\n",
      "440 tensor(1119.8699, grad_fn=<SumBackward0>)\n",
      "441 tensor(1112.4741, grad_fn=<SumBackward0>)\n",
      "442 tensor(1105.1394, grad_fn=<SumBackward0>)\n",
      "443 tensor(1097.8616, grad_fn=<SumBackward0>)\n",
      "444 tensor(1090.6478, grad_fn=<SumBackward0>)\n",
      "445 tensor(1083.4924, grad_fn=<SumBackward0>)\n",
      "446 tensor(1076.3955, grad_fn=<SumBackward0>)\n",
      "447 tensor(1069.3604, grad_fn=<SumBackward0>)\n",
      "448 tensor(1062.3790, grad_fn=<SumBackward0>)\n",
      "449 tensor(1055.4569, grad_fn=<SumBackward0>)\n",
      "450 tensor(1048.5927, grad_fn=<SumBackward0>)\n",
      "451 tensor(1041.7808, grad_fn=<SumBackward0>)\n",
      "452 tensor(1035.0278, grad_fn=<SumBackward0>)\n",
      "453 tensor(1028.3293, grad_fn=<SumBackward0>)\n",
      "454 tensor(1021.6860, grad_fn=<SumBackward0>)\n",
      "455 tensor(1015.0963, grad_fn=<SumBackward0>)\n",
      "456 tensor(1008.5637, grad_fn=<SumBackward0>)\n",
      "457 tensor(1002.0819, grad_fn=<SumBackward0>)\n",
      "458 tensor(995.6506, grad_fn=<SumBackward0>)\n",
      "459 tensor(989.2708, grad_fn=<SumBackward0>)\n",
      "460 tensor(982.9420, grad_fn=<SumBackward0>)\n",
      "461 tensor(976.6667, grad_fn=<SumBackward0>)\n",
      "462 tensor(970.4396, grad_fn=<SumBackward0>)\n",
      "463 tensor(964.2695, grad_fn=<SumBackward0>)\n",
      "464 tensor(958.1434, grad_fn=<SumBackward0>)\n",
      "465 tensor(952.0671, grad_fn=<SumBackward0>)\n",
      "466 tensor(946.0405, grad_fn=<SumBackward0>)\n",
      "467 tensor(940.0610, grad_fn=<SumBackward0>)\n",
      "468 tensor(934.1290, grad_fn=<SumBackward0>)\n",
      "469 tensor(928.2476, grad_fn=<SumBackward0>)\n",
      "470 tensor(922.4166, grad_fn=<SumBackward0>)\n",
      "471 tensor(916.6237, grad_fn=<SumBackward0>)\n",
      "472 tensor(910.8778, grad_fn=<SumBackward0>)\n",
      "473 tensor(905.1768, grad_fn=<SumBackward0>)\n",
      "474 tensor(899.5215, grad_fn=<SumBackward0>)\n",
      "475 tensor(893.9130, grad_fn=<SumBackward0>)\n",
      "476 tensor(888.3461, grad_fn=<SumBackward0>)\n",
      "477 tensor(882.8223, grad_fn=<SumBackward0>)\n",
      "478 tensor(877.3445, grad_fn=<SumBackward0>)\n",
      "479 tensor(871.9091, grad_fn=<SumBackward0>)\n",
      "480 tensor(866.5154, grad_fn=<SumBackward0>)\n",
      "481 tensor(861.1647, grad_fn=<SumBackward0>)\n",
      "482 tensor(855.8566, grad_fn=<SumBackward0>)\n",
      "483 tensor(850.5911, grad_fn=<SumBackward0>)\n",
      "484 tensor(845.3655, grad_fn=<SumBackward0>)\n",
      "485 tensor(840.1792, grad_fn=<SumBackward0>)\n",
      "486 tensor(835.0331, grad_fn=<SumBackward0>)\n",
      "487 tensor(829.9276, grad_fn=<SumBackward0>)\n",
      "488 tensor(824.8606, grad_fn=<SumBackward0>)\n",
      "489 tensor(819.8345, grad_fn=<SumBackward0>)\n",
      "490 tensor(814.8469, grad_fn=<SumBackward0>)\n",
      "491 tensor(809.8967, grad_fn=<SumBackward0>)\n",
      "492 tensor(804.9910, grad_fn=<SumBackward0>)\n",
      "493 tensor(800.1190, grad_fn=<SumBackward0>)\n",
      "494 tensor(795.2841, grad_fn=<SumBackward0>)\n",
      "495 tensor(790.4852, grad_fn=<SumBackward0>)\n",
      "496 tensor(785.7227, grad_fn=<SumBackward0>)\n",
      "497 tensor(780.9974, grad_fn=<SumBackward0>)\n",
      "498 tensor(776.3079, grad_fn=<SumBackward0>)\n",
      "499 tensor(771.6532, grad_fn=<SumBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# Autograd Version\n",
    "import torch\n",
    "\n",
    "# cpu or cuda\n",
    "device = torch.device('cpu')\n",
    "\n",
    "N, D_in, D_h, D_out = 100, 1000, 100, 10\n",
    "\n",
    "# generate training data\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# init the weights\n",
    "w1 = torch.randn(D_in, D_h, device=device, requires_grad=True)\n",
    "w2 = torch.randn(D_h, D_out, device=device, requires_grad=True)\n",
    "\n",
    "# training\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # forward inference\n",
    "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
    "    # calculate the loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss)\n",
    "    # use autograd to do the back-propagation\n",
    "    loss.backward()\n",
    "    # update the weights, prevent torch do autograd for this part\n",
    "    with torch.no_grad():\n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    "        # zero grads\n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 tensor(64044740., grad_fn=<SumBackward0>)\n",
      "1 tensor(1.8535e+08, grad_fn=<SumBackward0>)\n",
      "2 tensor(6.4584e+08, grad_fn=<SumBackward0>)\n",
      "3 tensor(4.4602e+08, grad_fn=<SumBackward0>)\n",
      "4 tensor(5713088., grad_fn=<SumBackward0>)\n",
      "5 tensor(4366853.5000, grad_fn=<SumBackward0>)\n",
      "6 tensor(3461743.7500, grad_fn=<SumBackward0>)\n",
      "7 tensor(2821221.5000, grad_fn=<SumBackward0>)\n",
      "8 tensor(2351187.7500, grad_fn=<SumBackward0>)\n",
      "9 tensor(1996174.5000, grad_fn=<SumBackward0>)\n",
      "10 tensor(1722102., grad_fn=<SumBackward0>)\n",
      "11 tensor(1506259.5000, grad_fn=<SumBackward0>)\n",
      "12 tensor(1333113.2500, grad_fn=<SumBackward0>)\n",
      "13 tensor(1191878.3750, grad_fn=<SumBackward0>)\n",
      "14 tensor(1074925.5000, grad_fn=<SumBackward0>)\n",
      "15 tensor(976912.8750, grad_fn=<SumBackward0>)\n",
      "16 tensor(893680.7500, grad_fn=<SumBackward0>)\n",
      "17 tensor(822140.6250, grad_fn=<SumBackward0>)\n",
      "18 tensor(760085., grad_fn=<SumBackward0>)\n",
      "19 tensor(705818.1875, grad_fn=<SumBackward0>)\n",
      "20 tensor(657946., grad_fn=<SumBackward0>)\n",
      "21 tensor(615333.1250, grad_fn=<SumBackward0>)\n",
      "22 tensor(577185.5000, grad_fn=<SumBackward0>)\n",
      "23 tensor(542774.4375, grad_fn=<SumBackward0>)\n",
      "24 tensor(511592.5000, grad_fn=<SumBackward0>)\n",
      "25 tensor(483204.9062, grad_fn=<SumBackward0>)\n",
      "26 tensor(457293.4375, grad_fn=<SumBackward0>)\n",
      "27 tensor(433559.3438, grad_fn=<SumBackward0>)\n",
      "28 tensor(411750.7500, grad_fn=<SumBackward0>)\n",
      "29 tensor(391623.8750, grad_fn=<SumBackward0>)\n",
      "30 tensor(372990.0312, grad_fn=<SumBackward0>)\n",
      "31 tensor(355683.2500, grad_fn=<SumBackward0>)\n",
      "32 tensor(339559.1875, grad_fn=<SumBackward0>)\n",
      "33 tensor(324515.8125, grad_fn=<SumBackward0>)\n",
      "34 tensor(310412.9062, grad_fn=<SumBackward0>)\n",
      "35 tensor(297201.3438, grad_fn=<SumBackward0>)\n",
      "36 tensor(284809.4688, grad_fn=<SumBackward0>)\n",
      "37 tensor(273166.6562, grad_fn=<SumBackward0>)\n",
      "38 tensor(262219.0625, grad_fn=<SumBackward0>)\n",
      "39 tensor(251883.4688, grad_fn=<SumBackward0>)\n",
      "40 tensor(242154.6562, grad_fn=<SumBackward0>)\n",
      "41 tensor(233005.5625, grad_fn=<SumBackward0>)\n",
      "42 tensor(224361.5312, grad_fn=<SumBackward0>)\n",
      "43 tensor(216182.2344, grad_fn=<SumBackward0>)\n",
      "44 tensor(208433.9375, grad_fn=<SumBackward0>)\n",
      "45 tensor(201090.5000, grad_fn=<SumBackward0>)\n",
      "46 tensor(194126.5625, grad_fn=<SumBackward0>)\n",
      "47 tensor(187517.5469, grad_fn=<SumBackward0>)\n",
      "48 tensor(181244.4375, grad_fn=<SumBackward0>)\n",
      "49 tensor(175285.4531, grad_fn=<SumBackward0>)\n",
      "50 tensor(169613.7812, grad_fn=<SumBackward0>)\n",
      "51 tensor(164211.4688, grad_fn=<SumBackward0>)\n",
      "52 tensor(159059.4844, grad_fn=<SumBackward0>)\n",
      "53 tensor(154154.0312, grad_fn=<SumBackward0>)\n",
      "54 tensor(149469.2812, grad_fn=<SumBackward0>)\n",
      "55 tensor(144987.5625, grad_fn=<SumBackward0>)\n",
      "56 tensor(140700.0625, grad_fn=<SumBackward0>)\n",
      "57 tensor(136596.5781, grad_fn=<SumBackward0>)\n",
      "58 tensor(132666.4219, grad_fn=<SumBackward0>)\n",
      "59 tensor(128898.6562, grad_fn=<SumBackward0>)\n",
      "60 tensor(125284.5938, grad_fn=<SumBackward0>)\n",
      "61 tensor(121818.2891, grad_fn=<SumBackward0>)\n",
      "62 tensor(118492.2188, grad_fn=<SumBackward0>)\n",
      "63 tensor(115296.9375, grad_fn=<SumBackward0>)\n",
      "64 tensor(112226.4297, grad_fn=<SumBackward0>)\n",
      "65 tensor(109273.6250, grad_fn=<SumBackward0>)\n",
      "66 tensor(106433.8828, grad_fn=<SumBackward0>)\n",
      "67 tensor(103699.2969, grad_fn=<SumBackward0>)\n",
      "68 tensor(101064.2734, grad_fn=<SumBackward0>)\n",
      "69 tensor(98527.0938, grad_fn=<SumBackward0>)\n",
      "70 tensor(96081.6562, grad_fn=<SumBackward0>)\n",
      "71 tensor(93726.3906, grad_fn=<SumBackward0>)\n",
      "72 tensor(91458.1172, grad_fn=<SumBackward0>)\n",
      "73 tensor(89270.4844, grad_fn=<SumBackward0>)\n",
      "74 tensor(87157.7344, grad_fn=<SumBackward0>)\n",
      "75 tensor(85116.9688, grad_fn=<SumBackward0>)\n",
      "76 tensor(83146.1953, grad_fn=<SumBackward0>)\n",
      "77 tensor(81241.9375, grad_fn=<SumBackward0>)\n",
      "78 tensor(79400.5547, grad_fn=<SumBackward0>)\n",
      "79 tensor(77617.9062, grad_fn=<SumBackward0>)\n",
      "80 tensor(75897.3203, grad_fn=<SumBackward0>)\n",
      "81 tensor(74231.1953, grad_fn=<SumBackward0>)\n",
      "82 tensor(72621.0625, grad_fn=<SumBackward0>)\n",
      "83 tensor(71062.0156, grad_fn=<SumBackward0>)\n",
      "84 tensor(69552.0938, grad_fn=<SumBackward0>)\n",
      "85 tensor(68088.6250, grad_fn=<SumBackward0>)\n",
      "86 tensor(66669.8984, grad_fn=<SumBackward0>)\n",
      "87 tensor(65293.7695, grad_fn=<SumBackward0>)\n",
      "88 tensor(63958.2930, grad_fn=<SumBackward0>)\n",
      "89 tensor(62661.5625, grad_fn=<SumBackward0>)\n",
      "90 tensor(61402.4531, grad_fn=<SumBackward0>)\n",
      "91 tensor(60179.4453, grad_fn=<SumBackward0>)\n",
      "92 tensor(58992.0469, grad_fn=<SumBackward0>)\n",
      "93 tensor(57839.2578, grad_fn=<SumBackward0>)\n",
      "94 tensor(56722.5898, grad_fn=<SumBackward0>)\n",
      "95 tensor(55639.5000, grad_fn=<SumBackward0>)\n",
      "96 tensor(54586.8203, grad_fn=<SumBackward0>)\n",
      "97 tensor(53563.9258, grad_fn=<SumBackward0>)\n",
      "98 tensor(52569.7734, grad_fn=<SumBackward0>)\n",
      "99 tensor(51602.5469, grad_fn=<SumBackward0>)\n",
      "100 tensor(50662.2109, grad_fn=<SumBackward0>)\n",
      "101 tensor(49750.2578, grad_fn=<SumBackward0>)\n",
      "102 tensor(48866.2578, grad_fn=<SumBackward0>)\n",
      "103 tensor(48006.3359, grad_fn=<SumBackward0>)\n",
      "104 tensor(47168.1953, grad_fn=<SumBackward0>)\n",
      "105 tensor(46352.1133, grad_fn=<SumBackward0>)\n",
      "106 tensor(45557.5625, grad_fn=<SumBackward0>)\n",
      "107 tensor(44782.7500, grad_fn=<SumBackward0>)\n",
      "108 tensor(44026.7891, grad_fn=<SumBackward0>)\n",
      "109 tensor(43289.7383, grad_fn=<SumBackward0>)\n",
      "110 tensor(42570.6211, grad_fn=<SumBackward0>)\n",
      "111 tensor(41868.4492, grad_fn=<SumBackward0>)\n",
      "112 tensor(41182.7109, grad_fn=<SumBackward0>)\n",
      "113 tensor(40513.2734, grad_fn=<SumBackward0>)\n",
      "114 tensor(39859.8594, grad_fn=<SumBackward0>)\n",
      "115 tensor(39221.5078, grad_fn=<SumBackward0>)\n",
      "116 tensor(38597.8242, grad_fn=<SumBackward0>)\n",
      "117 tensor(37988.2148, grad_fn=<SumBackward0>)\n",
      "118 tensor(37392.8125, grad_fn=<SumBackward0>)\n",
      "119 tensor(36811.3828, grad_fn=<SumBackward0>)\n",
      "120 tensor(36243.0625, grad_fn=<SumBackward0>)\n",
      "121 tensor(35687.3711, grad_fn=<SumBackward0>)\n",
      "122 tensor(35143.1367, grad_fn=<SumBackward0>)\n",
      "123 tensor(34610.8008, grad_fn=<SumBackward0>)\n",
      "124 tensor(34090.0234, grad_fn=<SumBackward0>)\n",
      "125 tensor(33580.6719, grad_fn=<SumBackward0>)\n",
      "126 tensor(33082.1328, grad_fn=<SumBackward0>)\n",
      "127 tensor(32594.1738, grad_fn=<SumBackward0>)\n",
      "128 tensor(32117.2109, grad_fn=<SumBackward0>)\n",
      "129 tensor(31649.0938, grad_fn=<SumBackward0>)\n",
      "130 tensor(31190.5938, grad_fn=<SumBackward0>)\n",
      "131 tensor(30741.9766, grad_fn=<SumBackward0>)\n",
      "132 tensor(30302.6094, grad_fn=<SumBackward0>)\n",
      "133 tensor(29872.2031, grad_fn=<SumBackward0>)\n",
      "134 tensor(29450.5859, grad_fn=<SumBackward0>)\n",
      "135 tensor(29039.3398, grad_fn=<SumBackward0>)\n",
      "136 tensor(28636.9922, grad_fn=<SumBackward0>)\n",
      "137 tensor(28243.8047, grad_fn=<SumBackward0>)\n",
      "138 tensor(27858.4609, grad_fn=<SumBackward0>)\n",
      "139 tensor(27480.8809, grad_fn=<SumBackward0>)\n",
      "140 tensor(27111.0195, grad_fn=<SumBackward0>)\n",
      "141 tensor(26748.4688, grad_fn=<SumBackward0>)\n",
      "142 tensor(26393.1602, grad_fn=<SumBackward0>)\n",
      "143 tensor(26044.6523, grad_fn=<SumBackward0>)\n",
      "144 tensor(25702.7422, grad_fn=<SumBackward0>)\n",
      "145 tensor(25367.3672, grad_fn=<SumBackward0>)\n",
      "146 tensor(25038.5977, grad_fn=<SumBackward0>)\n",
      "147 tensor(24715.9180, grad_fn=<SumBackward0>)\n",
      "148 tensor(24399.4336, grad_fn=<SumBackward0>)\n",
      "149 tensor(24088.7695, grad_fn=<SumBackward0>)\n",
      "150 tensor(23783.8359, grad_fn=<SumBackward0>)\n",
      "151 tensor(23484.7422, grad_fn=<SumBackward0>)\n",
      "152 tensor(23190.8477, grad_fn=<SumBackward0>)\n",
      "153 tensor(22902.0977, grad_fn=<SumBackward0>)\n",
      "154 tensor(22618.6035, grad_fn=<SumBackward0>)\n",
      "155 tensor(22340.2832, grad_fn=<SumBackward0>)\n",
      "156 tensor(22066.7949, grad_fn=<SumBackward0>)\n",
      "157 tensor(21798.0605, grad_fn=<SumBackward0>)\n",
      "158 tensor(21534.0352, grad_fn=<SumBackward0>)\n",
      "159 tensor(21274.7148, grad_fn=<SumBackward0>)\n",
      "160 tensor(21019.9258, grad_fn=<SumBackward0>)\n",
      "161 tensor(20769.3730, grad_fn=<SumBackward0>)\n",
      "162 tensor(20523.0156, grad_fn=<SumBackward0>)\n",
      "163 tensor(20280.9375, grad_fn=<SumBackward0>)\n",
      "164 tensor(20043.0742, grad_fn=<SumBackward0>)\n",
      "165 tensor(19809.2266, grad_fn=<SumBackward0>)\n",
      "166 tensor(19577.4648, grad_fn=<SumBackward0>)\n",
      "167 tensor(19349.6660, grad_fn=<SumBackward0>)\n",
      "168 tensor(19125.7266, grad_fn=<SumBackward0>)\n",
      "169 tensor(18905.5137, grad_fn=<SumBackward0>)\n",
      "170 tensor(18689.0273, grad_fn=<SumBackward0>)\n",
      "171 tensor(18476.0566, grad_fn=<SumBackward0>)\n",
      "172 tensor(18266.5469, grad_fn=<SumBackward0>)\n",
      "173 tensor(18060.3926, grad_fn=<SumBackward0>)\n",
      "174 tensor(17857.5352, grad_fn=<SumBackward0>)\n",
      "175 tensor(17658.0391, grad_fn=<SumBackward0>)\n",
      "176 tensor(17461.8906, grad_fn=<SumBackward0>)\n",
      "177 tensor(17268.7559, grad_fn=<SumBackward0>)\n",
      "178 tensor(17078.7227, grad_fn=<SumBackward0>)\n",
      "179 tensor(16891.7461, grad_fn=<SumBackward0>)\n",
      "180 tensor(16707.7500, grad_fn=<SumBackward0>)\n",
      "181 tensor(16526.4043, grad_fn=<SumBackward0>)\n",
      "182 tensor(16347.9717, grad_fn=<SumBackward0>)\n",
      "183 tensor(16172.4541, grad_fn=<SumBackward0>)\n",
      "184 tensor(15999.6387, grad_fn=<SumBackward0>)\n",
      "185 tensor(15829.4932, grad_fn=<SumBackward0>)\n",
      "186 tensor(15661.9434, grad_fn=<SumBackward0>)\n",
      "187 tensor(15496.9355, grad_fn=<SumBackward0>)\n",
      "188 tensor(15334.5342, grad_fn=<SumBackward0>)\n",
      "189 tensor(15174.5576, grad_fn=<SumBackward0>)\n",
      "190 tensor(15016.8672, grad_fn=<SumBackward0>)\n",
      "191 tensor(14861.5449, grad_fn=<SumBackward0>)\n",
      "192 tensor(14708.6367, grad_fn=<SumBackward0>)\n",
      "193 tensor(14557.8232, grad_fn=<SumBackward0>)\n",
      "194 tensor(14409.2764, grad_fn=<SumBackward0>)\n",
      "195 tensor(14262.8633, grad_fn=<SumBackward0>)\n",
      "196 tensor(14118.5996, grad_fn=<SumBackward0>)\n",
      "197 tensor(13976.4219, grad_fn=<SumBackward0>)\n",
      "198 tensor(13836.2441, grad_fn=<SumBackward0>)\n",
      "199 tensor(13697.9443, grad_fn=<SumBackward0>)\n",
      "200 tensor(13560.4922, grad_fn=<SumBackward0>)\n",
      "201 tensor(13425.0684, grad_fn=<SumBackward0>)\n",
      "202 tensor(13291.5928, grad_fn=<SumBackward0>)\n",
      "203 tensor(13160.0586, grad_fn=<SumBackward0>)\n",
      "204 tensor(13030.4766, grad_fn=<SumBackward0>)\n",
      "205 tensor(12902.7959, grad_fn=<SumBackward0>)\n",
      "206 tensor(12776.9619, grad_fn=<SumBackward0>)\n",
      "207 tensor(12652.8438, grad_fn=<SumBackward0>)\n",
      "208 tensor(12527.8086, grad_fn=<SumBackward0>)\n",
      "209 tensor(12404.5186, grad_fn=<SumBackward0>)\n",
      "210 tensor(12282.9209, grad_fn=<SumBackward0>)\n",
      "211 tensor(12163.1533, grad_fn=<SumBackward0>)\n",
      "212 tensor(12045.0273, grad_fn=<SumBackward0>)\n",
      "213 tensor(11928.5234, grad_fn=<SumBackward0>)\n",
      "214 tensor(11813.6602, grad_fn=<SumBackward0>)\n",
      "215 tensor(11700.3887, grad_fn=<SumBackward0>)\n",
      "216 tensor(11588.6260, grad_fn=<SumBackward0>)\n",
      "217 tensor(11478.3525, grad_fn=<SumBackward0>)\n",
      "218 tensor(11369.6270, grad_fn=<SumBackward0>)\n",
      "219 tensor(11262.3965, grad_fn=<SumBackward0>)\n",
      "220 tensor(11156.5703, grad_fn=<SumBackward0>)\n",
      "221 tensor(11052.2471, grad_fn=<SumBackward0>)\n",
      "222 tensor(10949.3438, grad_fn=<SumBackward0>)\n",
      "223 tensor(10847.8086, grad_fn=<SumBackward0>)\n",
      "224 tensor(10747.5967, grad_fn=<SumBackward0>)\n",
      "225 tensor(10648.6562, grad_fn=<SumBackward0>)\n",
      "226 tensor(10551.0410, grad_fn=<SumBackward0>)\n",
      "227 tensor(10454.7383, grad_fn=<SumBackward0>)\n",
      "228 tensor(10359.6250, grad_fn=<SumBackward0>)\n",
      "229 tensor(10265.7676, grad_fn=<SumBackward0>)\n",
      "230 tensor(10173.1270, grad_fn=<SumBackward0>)\n",
      "231 tensor(10081.6494, grad_fn=<SumBackward0>)\n",
      "232 tensor(9991.3457, grad_fn=<SumBackward0>)\n",
      "233 tensor(9902.1641, grad_fn=<SumBackward0>)\n",
      "234 tensor(9814.1797, grad_fn=<SumBackward0>)\n",
      "235 tensor(9727.3057, grad_fn=<SumBackward0>)\n",
      "236 tensor(9641.4688, grad_fn=<SumBackward0>)\n",
      "237 tensor(9556.7314, grad_fn=<SumBackward0>)\n",
      "238 tensor(9473.0654, grad_fn=<SumBackward0>)\n",
      "239 tensor(9390.4287, grad_fn=<SumBackward0>)\n",
      "240 tensor(9308.8145, grad_fn=<SumBackward0>)\n",
      "241 tensor(9228.2061, grad_fn=<SumBackward0>)\n",
      "242 tensor(9148.5820, grad_fn=<SumBackward0>)\n",
      "243 tensor(9069.9160, grad_fn=<SumBackward0>)\n",
      "244 tensor(8992.2197, grad_fn=<SumBackward0>)\n",
      "245 tensor(8915.4395, grad_fn=<SumBackward0>)\n",
      "246 tensor(8839.6299, grad_fn=<SumBackward0>)\n",
      "247 tensor(8764.7598, grad_fn=<SumBackward0>)\n",
      "248 tensor(8690.7812, grad_fn=<SumBackward0>)\n",
      "249 tensor(8617.7129, grad_fn=<SumBackward0>)\n",
      "250 tensor(8545.5254, grad_fn=<SumBackward0>)\n",
      "251 tensor(8474.1699, grad_fn=<SumBackward0>)\n",
      "252 tensor(8403.6406, grad_fn=<SumBackward0>)\n",
      "253 tensor(8333.9473, grad_fn=<SumBackward0>)\n",
      "254 tensor(8265.0977, grad_fn=<SumBackward0>)\n",
      "255 tensor(8197.0293, grad_fn=<SumBackward0>)\n",
      "256 tensor(8129.7749, grad_fn=<SumBackward0>)\n",
      "257 tensor(8063.3188, grad_fn=<SumBackward0>)\n",
      "258 tensor(7997.6050, grad_fn=<SumBackward0>)\n",
      "259 tensor(7932.6421, grad_fn=<SumBackward0>)\n",
      "260 tensor(7868.5005, grad_fn=<SumBackward0>)\n",
      "261 tensor(7805.1099, grad_fn=<SumBackward0>)\n",
      "262 tensor(7742.4199, grad_fn=<SumBackward0>)\n",
      "263 tensor(7680.4795, grad_fn=<SumBackward0>)\n",
      "264 tensor(7619.2783, grad_fn=<SumBackward0>)\n",
      "265 tensor(7558.7773, grad_fn=<SumBackward0>)\n",
      "266 tensor(7498.9795, grad_fn=<SumBackward0>)\n",
      "267 tensor(7439.8936, grad_fn=<SumBackward0>)\n",
      "268 tensor(7381.4844, grad_fn=<SumBackward0>)\n",
      "269 tensor(7323.6802, grad_fn=<SumBackward0>)\n",
      "270 tensor(7266.5308, grad_fn=<SumBackward0>)\n",
      "271 tensor(7210.0322, grad_fn=<SumBackward0>)\n",
      "272 tensor(7154.1670, grad_fn=<SumBackward0>)\n",
      "273 tensor(7098.8940, grad_fn=<SumBackward0>)\n",
      "274 tensor(7044.3096, grad_fn=<SumBackward0>)\n",
      "275 tensor(6990.3525, grad_fn=<SumBackward0>)\n",
      "276 tensor(6936.9980, grad_fn=<SumBackward0>)\n",
      "277 tensor(6884.2236, grad_fn=<SumBackward0>)\n",
      "278 tensor(6832.0190, grad_fn=<SumBackward0>)\n",
      "279 tensor(6780.3926, grad_fn=<SumBackward0>)\n",
      "280 tensor(6729.3052, grad_fn=<SumBackward0>)\n",
      "281 tensor(6678.7666, grad_fn=<SumBackward0>)\n",
      "282 tensor(6628.7690, grad_fn=<SumBackward0>)\n",
      "283 tensor(6579.3384, grad_fn=<SumBackward0>)\n",
      "284 tensor(6530.4141, grad_fn=<SumBackward0>)\n",
      "285 tensor(6482.0332, grad_fn=<SumBackward0>)\n",
      "286 tensor(6434.1733, grad_fn=<SumBackward0>)\n",
      "287 tensor(6386.8154, grad_fn=<SumBackward0>)\n",
      "288 tensor(6339.9365, grad_fn=<SumBackward0>)\n",
      "289 tensor(6293.5479, grad_fn=<SumBackward0>)\n",
      "290 tensor(6247.6567, grad_fn=<SumBackward0>)\n",
      "291 tensor(6202.2715, grad_fn=<SumBackward0>)\n",
      "292 tensor(6157.3184, grad_fn=<SumBackward0>)\n",
      "293 tensor(6112.8301, grad_fn=<SumBackward0>)\n",
      "294 tensor(6068.8208, grad_fn=<SumBackward0>)\n",
      "295 tensor(6025.2500, grad_fn=<SumBackward0>)\n",
      "296 tensor(5982.1230, grad_fn=<SumBackward0>)\n",
      "297 tensor(5939.4766, grad_fn=<SumBackward0>)\n",
      "298 tensor(5897.3101, grad_fn=<SumBackward0>)\n",
      "299 tensor(5855.5693, grad_fn=<SumBackward0>)\n",
      "300 tensor(5814.2578, grad_fn=<SumBackward0>)\n",
      "301 tensor(5773.3560, grad_fn=<SumBackward0>)\n",
      "302 tensor(5732.8721, grad_fn=<SumBackward0>)\n",
      "303 tensor(5692.8047, grad_fn=<SumBackward0>)\n",
      "304 tensor(5653.1450, grad_fn=<SumBackward0>)\n",
      "305 tensor(5613.8633, grad_fn=<SumBackward0>)\n",
      "306 tensor(5574.9897, grad_fn=<SumBackward0>)\n",
      "307 tensor(5536.4971, grad_fn=<SumBackward0>)\n",
      "308 tensor(5498.3843, grad_fn=<SumBackward0>)\n",
      "309 tensor(5460.6494, grad_fn=<SumBackward0>)\n",
      "310 tensor(5423.2671, grad_fn=<SumBackward0>)\n",
      "311 tensor(5386.2607, grad_fn=<SumBackward0>)\n",
      "312 tensor(5349.6055, grad_fn=<SumBackward0>)\n",
      "313 tensor(5313.3350, grad_fn=<SumBackward0>)\n",
      "314 tensor(5277.3984, grad_fn=<SumBackward0>)\n",
      "315 tensor(5241.7993, grad_fn=<SumBackward0>)\n",
      "316 tensor(5206.5703, grad_fn=<SumBackward0>)\n",
      "317 tensor(5171.6777, grad_fn=<SumBackward0>)\n",
      "318 tensor(5137.1050, grad_fn=<SumBackward0>)\n",
      "319 tensor(5102.8721, grad_fn=<SumBackward0>)\n",
      "320 tensor(5069.0059, grad_fn=<SumBackward0>)\n",
      "321 tensor(5035.4668, grad_fn=<SumBackward0>)\n",
      "322 tensor(5002.2529, grad_fn=<SumBackward0>)\n",
      "323 tensor(4969.3613, grad_fn=<SumBackward0>)\n",
      "324 tensor(4936.7910, grad_fn=<SumBackward0>)\n",
      "325 tensor(4904.5410, grad_fn=<SumBackward0>)\n",
      "326 tensor(4872.5664, grad_fn=<SumBackward0>)\n",
      "327 tensor(4840.8906, grad_fn=<SumBackward0>)\n",
      "328 tensor(4809.5210, grad_fn=<SumBackward0>)\n",
      "329 tensor(4778.4541, grad_fn=<SumBackward0>)\n",
      "330 tensor(4747.6787, grad_fn=<SumBackward0>)\n",
      "331 tensor(4717.1724, grad_fn=<SumBackward0>)\n",
      "332 tensor(4686.9634, grad_fn=<SumBackward0>)\n",
      "333 tensor(4657.0054, grad_fn=<SumBackward0>)\n",
      "334 tensor(4627.3320, grad_fn=<SumBackward0>)\n",
      "335 tensor(4597.9463, grad_fn=<SumBackward0>)\n",
      "336 tensor(4568.8091, grad_fn=<SumBackward0>)\n",
      "337 tensor(4539.9351, grad_fn=<SumBackward0>)\n",
      "338 tensor(4511.3291, grad_fn=<SumBackward0>)\n",
      "339 tensor(4482.9829, grad_fn=<SumBackward0>)\n",
      "340 tensor(4454.8975, grad_fn=<SumBackward0>)\n",
      "341 tensor(4427.0908, grad_fn=<SumBackward0>)\n",
      "342 tensor(4399.4888, grad_fn=<SumBackward0>)\n",
      "343 tensor(4372.1562, grad_fn=<SumBackward0>)\n",
      "344 tensor(4345.0649, grad_fn=<SumBackward0>)\n",
      "345 tensor(4318.2021, grad_fn=<SumBackward0>)\n",
      "346 tensor(4291.5757, grad_fn=<SumBackward0>)\n",
      "347 tensor(4265.1924, grad_fn=<SumBackward0>)\n",
      "348 tensor(4239.0420, grad_fn=<SumBackward0>)\n",
      "349 tensor(4213.1182, grad_fn=<SumBackward0>)\n",
      "350 tensor(4187.4277, grad_fn=<SumBackward0>)\n",
      "351 tensor(4161.9580, grad_fn=<SumBackward0>)\n",
      "352 tensor(4136.7080, grad_fn=<SumBackward0>)\n",
      "353 tensor(4111.6860, grad_fn=<SumBackward0>)\n",
      "354 tensor(4086.9236, grad_fn=<SumBackward0>)\n",
      "355 tensor(4062.3872, grad_fn=<SumBackward0>)\n",
      "356 tensor(4038.0552, grad_fn=<SumBackward0>)\n",
      "357 tensor(4013.9397, grad_fn=<SumBackward0>)\n",
      "358 tensor(3990.0378, grad_fn=<SumBackward0>)\n",
      "359 tensor(3966.3442, grad_fn=<SumBackward0>)\n",
      "360 tensor(3942.8550, grad_fn=<SumBackward0>)\n",
      "361 tensor(3919.5681, grad_fn=<SumBackward0>)\n",
      "362 tensor(3896.4807, grad_fn=<SumBackward0>)\n",
      "363 tensor(3873.5984, grad_fn=<SumBackward0>)\n",
      "364 tensor(3850.8896, grad_fn=<SumBackward0>)\n",
      "365 tensor(3828.3804, grad_fn=<SumBackward0>)\n",
      "366 tensor(3806.0610, grad_fn=<SumBackward0>)\n",
      "367 tensor(3783.9387, grad_fn=<SumBackward0>)\n",
      "368 tensor(3761.9932, grad_fn=<SumBackward0>)\n",
      "369 tensor(3740.2397, grad_fn=<SumBackward0>)\n",
      "370 tensor(3718.6582, grad_fn=<SumBackward0>)\n",
      "371 tensor(3697.2593, grad_fn=<SumBackward0>)\n",
      "372 tensor(3676.0542, grad_fn=<SumBackward0>)\n",
      "373 tensor(3655.0273, grad_fn=<SumBackward0>)\n",
      "374 tensor(3634.1489, grad_fn=<SumBackward0>)\n",
      "375 tensor(3613.4536, grad_fn=<SumBackward0>)\n",
      "376 tensor(3592.9248, grad_fn=<SumBackward0>)\n",
      "377 tensor(3572.5737, grad_fn=<SumBackward0>)\n",
      "378 tensor(3552.3943, grad_fn=<SumBackward0>)\n",
      "379 tensor(3532.3772, grad_fn=<SumBackward0>)\n",
      "380 tensor(3512.5154, grad_fn=<SumBackward0>)\n",
      "381 tensor(3492.8113, grad_fn=<SumBackward0>)\n",
      "382 tensor(3473.2729, grad_fn=<SumBackward0>)\n",
      "383 tensor(3453.8953, grad_fn=<SumBackward0>)\n",
      "384 tensor(3434.6665, grad_fn=<SumBackward0>)\n",
      "385 tensor(3415.5918, grad_fn=<SumBackward0>)\n",
      "386 tensor(3396.6807, grad_fn=<SumBackward0>)\n",
      "387 tensor(3377.9285, grad_fn=<SumBackward0>)\n",
      "388 tensor(3359.3159, grad_fn=<SumBackward0>)\n",
      "389 tensor(3340.8423, grad_fn=<SumBackward0>)\n",
      "390 tensor(3322.5146, grad_fn=<SumBackward0>)\n",
      "391 tensor(3304.3408, grad_fn=<SumBackward0>)\n",
      "392 tensor(3286.3030, grad_fn=<SumBackward0>)\n",
      "393 tensor(3268.4116, grad_fn=<SumBackward0>)\n",
      "394 tensor(3250.6631, grad_fn=<SumBackward0>)\n",
      "395 tensor(3233.0530, grad_fn=<SumBackward0>)\n",
      "396 tensor(3215.5803, grad_fn=<SumBackward0>)\n",
      "397 tensor(3198.2688, grad_fn=<SumBackward0>)\n",
      "398 tensor(3181.1006, grad_fn=<SumBackward0>)\n",
      "399 tensor(3164.0706, grad_fn=<SumBackward0>)\n",
      "400 tensor(3147.1653, grad_fn=<SumBackward0>)\n",
      "401 tensor(3130.3999, grad_fn=<SumBackward0>)\n",
      "402 tensor(3113.7700, grad_fn=<SumBackward0>)\n",
      "403 tensor(3097.2549, grad_fn=<SumBackward0>)\n",
      "404 tensor(3080.8677, grad_fn=<SumBackward0>)\n",
      "405 tensor(3064.6064, grad_fn=<SumBackward0>)\n",
      "406 tensor(3048.4761, grad_fn=<SumBackward0>)\n",
      "407 tensor(3032.4700, grad_fn=<SumBackward0>)\n",
      "408 tensor(3016.5835, grad_fn=<SumBackward0>)\n",
      "409 tensor(3000.8096, grad_fn=<SumBackward0>)\n",
      "410 tensor(2985.1626, grad_fn=<SumBackward0>)\n",
      "411 tensor(2969.6260, grad_fn=<SumBackward0>)\n",
      "412 tensor(2954.2122, grad_fn=<SumBackward0>)\n",
      "413 tensor(2938.9124, grad_fn=<SumBackward0>)\n",
      "414 tensor(2923.7317, grad_fn=<SumBackward0>)\n",
      "415 tensor(2908.6655, grad_fn=<SumBackward0>)\n",
      "416 tensor(2893.7151, grad_fn=<SumBackward0>)\n",
      "417 tensor(2878.8796, grad_fn=<SumBackward0>)\n",
      "418 tensor(2864.1401, grad_fn=<SumBackward0>)\n",
      "419 tensor(2849.5134, grad_fn=<SumBackward0>)\n",
      "420 tensor(2835.0061, grad_fn=<SumBackward0>)\n",
      "421 tensor(2820.6033, grad_fn=<SumBackward0>)\n",
      "422 tensor(2806.3057, grad_fn=<SumBackward0>)\n",
      "423 tensor(2792.1147, grad_fn=<SumBackward0>)\n",
      "424 tensor(2778.0293, grad_fn=<SumBackward0>)\n",
      "425 tensor(2764.0601, grad_fn=<SumBackward0>)\n",
      "426 tensor(2750.1919, grad_fn=<SumBackward0>)\n",
      "427 tensor(2736.4216, grad_fn=<SumBackward0>)\n",
      "428 tensor(2722.7505, grad_fn=<SumBackward0>)\n",
      "429 tensor(2709.1787, grad_fn=<SumBackward0>)\n",
      "430 tensor(2695.7085, grad_fn=<SumBackward0>)\n",
      "431 tensor(2682.3416, grad_fn=<SumBackward0>)\n",
      "432 tensor(2669.0618, grad_fn=<SumBackward0>)\n",
      "433 tensor(2655.8877, grad_fn=<SumBackward0>)\n",
      "434 tensor(2642.7969, grad_fn=<SumBackward0>)\n",
      "435 tensor(2629.8040, grad_fn=<SumBackward0>)\n",
      "436 tensor(2616.8982, grad_fn=<SumBackward0>)\n",
      "437 tensor(2604.0913, grad_fn=<SumBackward0>)\n",
      "438 tensor(2591.3689, grad_fn=<SumBackward0>)\n",
      "439 tensor(2578.7473, grad_fn=<SumBackward0>)\n",
      "440 tensor(2566.2217, grad_fn=<SumBackward0>)\n",
      "441 tensor(2553.7837, grad_fn=<SumBackward0>)\n",
      "442 tensor(2541.4355, grad_fn=<SumBackward0>)\n",
      "443 tensor(2529.1721, grad_fn=<SumBackward0>)\n",
      "444 tensor(2516.9956, grad_fn=<SumBackward0>)\n",
      "445 tensor(2504.9072, grad_fn=<SumBackward0>)\n",
      "446 tensor(2492.8955, grad_fn=<SumBackward0>)\n",
      "447 tensor(2480.9680, grad_fn=<SumBackward0>)\n",
      "448 tensor(2469.1235, grad_fn=<SumBackward0>)\n",
      "449 tensor(2457.3696, grad_fn=<SumBackward0>)\n",
      "450 tensor(2445.6973, grad_fn=<SumBackward0>)\n",
      "451 tensor(2434.0977, grad_fn=<SumBackward0>)\n",
      "452 tensor(2422.5752, grad_fn=<SumBackward0>)\n",
      "453 tensor(2411.1333, grad_fn=<SumBackward0>)\n",
      "454 tensor(2399.7822, grad_fn=<SumBackward0>)\n",
      "455 tensor(2388.5005, grad_fn=<SumBackward0>)\n",
      "456 tensor(2377.2974, grad_fn=<SumBackward0>)\n",
      "457 tensor(2366.1689, grad_fn=<SumBackward0>)\n",
      "458 tensor(2355.1133, grad_fn=<SumBackward0>)\n",
      "459 tensor(2344.1328, grad_fn=<SumBackward0>)\n",
      "460 tensor(2333.2258, grad_fn=<SumBackward0>)\n",
      "461 tensor(2322.3914, grad_fn=<SumBackward0>)\n",
      "462 tensor(2311.6299, grad_fn=<SumBackward0>)\n",
      "463 tensor(2300.9456, grad_fn=<SumBackward0>)\n",
      "464 tensor(2290.3328, grad_fn=<SumBackward0>)\n",
      "465 tensor(2279.7876, grad_fn=<SumBackward0>)\n",
      "466 tensor(2269.3203, grad_fn=<SumBackward0>)\n",
      "467 tensor(2258.9272, grad_fn=<SumBackward0>)\n",
      "468 tensor(2248.6445, grad_fn=<SumBackward0>)\n",
      "469 tensor(2238.4290, grad_fn=<SumBackward0>)\n",
      "470 tensor(2228.2759, grad_fn=<SumBackward0>)\n",
      "471 tensor(2218.1948, grad_fn=<SumBackward0>)\n",
      "472 tensor(2208.1816, grad_fn=<SumBackward0>)\n",
      "473 tensor(2198.2358, grad_fn=<SumBackward0>)\n",
      "474 tensor(2188.3533, grad_fn=<SumBackward0>)\n",
      "475 tensor(2178.5298, grad_fn=<SumBackward0>)\n",
      "476 tensor(2168.7732, grad_fn=<SumBackward0>)\n",
      "477 tensor(2159.0830, grad_fn=<SumBackward0>)\n",
      "478 tensor(2149.4548, grad_fn=<SumBackward0>)\n",
      "479 tensor(2139.8892, grad_fn=<SumBackward0>)\n",
      "480 tensor(2130.3857, grad_fn=<SumBackward0>)\n",
      "481 tensor(2120.9482, grad_fn=<SumBackward0>)\n",
      "482 tensor(2111.5693, grad_fn=<SumBackward0>)\n",
      "483 tensor(2102.2529, grad_fn=<SumBackward0>)\n",
      "484 tensor(2093.0012, grad_fn=<SumBackward0>)\n",
      "485 tensor(2083.7983, grad_fn=<SumBackward0>)\n",
      "486 tensor(2074.6548, grad_fn=<SumBackward0>)\n",
      "487 tensor(2065.5684, grad_fn=<SumBackward0>)\n",
      "488 tensor(2056.5449, grad_fn=<SumBackward0>)\n",
      "489 tensor(2047.5790, grad_fn=<SumBackward0>)\n",
      "490 tensor(2038.6699, grad_fn=<SumBackward0>)\n",
      "491 tensor(2029.8165, grad_fn=<SumBackward0>)\n",
      "492 tensor(2021.0245, grad_fn=<SumBackward0>)\n",
      "493 tensor(2012.2842, grad_fn=<SumBackward0>)\n",
      "494 tensor(2003.5966, grad_fn=<SumBackward0>)\n",
      "495 tensor(1994.9669, grad_fn=<SumBackward0>)\n",
      "496 tensor(1986.3889, grad_fn=<SumBackward0>)\n",
      "497 tensor(1977.8611, grad_fn=<SumBackward0>)\n",
      "498 tensor(1969.3894, grad_fn=<SumBackward0>)\n",
      "499 tensor(1960.9716, grad_fn=<SumBackward0>)\n"
     ]
    }
   ],
   "source": [
    "# MyReLU Version\n",
    "import torch\n",
    "from my_relu import MyReLU\n",
    "\n",
    "# cpu or cuda\n",
    "device = torch.device('cpu')\n",
    "\n",
    "N, D_in, D_h, D_out = 100, 1000, 100, 10\n",
    "\n",
    "# generate training data\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# init the weights\n",
    "w1 = torch.randn(D_in, D_h, device=device, requires_grad=True)\n",
    "w2 = torch.randn(D_h, D_out, device=device, requires_grad=True)\n",
    "\n",
    "# training\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    # forward inference, with MyReLU\n",
    "    y_pred = MyReLU.apply(x.mm(w1)).mm(w2)\n",
    "    # calculate the loss\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss)\n",
    "    # use autograd to do the back-propagation\n",
    "    # gonna call MyReLU.backward()\n",
    "    loss.backward()\n",
    "    # update the weights, prevent torch do autograd for this part\n",
    "    with torch.no_grad():\n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    "        # zero grads\n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 1055.11572265625\n",
      "1 979.7573852539062\n",
      "2 914.0875244140625\n",
      "3 856.3018798828125\n",
      "4 804.7339477539062\n",
      "5 758.6033325195312\n",
      "6 716.502685546875\n",
      "7 677.8682861328125\n",
      "8 641.9003295898438\n",
      "9 608.1055908203125\n",
      "10 576.0472412109375\n",
      "11 545.6200561523438\n",
      "12 516.5584716796875\n",
      "13 488.8470153808594\n",
      "14 462.3072814941406\n",
      "15 436.818603515625\n",
      "16 412.47161865234375\n",
      "17 389.1689453125\n",
      "18 366.92864990234375\n",
      "19 345.71099853515625\n",
      "20 325.39764404296875\n",
      "21 306.02178955078125\n",
      "22 287.5928039550781\n",
      "23 270.0403137207031\n",
      "24 253.38345336914062\n",
      "25 237.55694580078125\n",
      "26 222.5810089111328\n",
      "27 208.38523864746094\n",
      "28 194.99539184570312\n",
      "29 182.37692260742188\n",
      "30 170.48968505859375\n",
      "31 159.3143310546875\n",
      "32 148.81051635742188\n",
      "33 138.978271484375\n",
      "34 129.75088500976562\n",
      "35 121.11564636230469\n",
      "36 113.04875183105469\n",
      "37 105.51568603515625\n",
      "38 98.45283508300781\n",
      "39 91.86031341552734\n",
      "40 85.70729064941406\n",
      "41 79.97239685058594\n",
      "42 74.62997436523438\n",
      "43 69.65254211425781\n",
      "44 65.01960754394531\n",
      "45 60.70447540283203\n",
      "46 56.688720703125\n",
      "47 52.95580291748047\n",
      "48 49.482505798339844\n",
      "49 46.24998092651367\n",
      "50 43.23950958251953\n",
      "51 40.437862396240234\n",
      "52 37.832069396972656\n",
      "53 35.4056396484375\n",
      "54 33.15058135986328\n",
      "55 31.045486450195312\n",
      "56 29.082117080688477\n",
      "57 27.250375747680664\n",
      "58 25.54108238220215\n",
      "59 23.946943283081055\n",
      "60 22.463455200195312\n",
      "61 21.078937530517578\n",
      "62 19.78757095336914\n",
      "63 18.58313751220703\n",
      "64 17.45647430419922\n",
      "65 16.403644561767578\n",
      "66 15.418587684631348\n",
      "67 14.497918128967285\n",
      "68 13.635720252990723\n",
      "69 12.830231666564941\n",
      "70 12.075481414794922\n",
      "71 11.36806583404541\n",
      "72 10.7061185836792\n",
      "73 10.086404800415039\n",
      "74 9.50631332397461\n",
      "75 8.961125373840332\n",
      "76 8.450262069702148\n",
      "77 7.97098445892334\n",
      "78 7.521007537841797\n",
      "79 7.097360610961914\n",
      "80 6.699847221374512\n",
      "81 6.326662063598633\n",
      "82 5.975793361663818\n",
      "83 5.646134853363037\n",
      "84 5.336431503295898\n",
      "85 5.045056343078613\n",
      "86 4.7697296142578125\n",
      "87 4.51077938079834\n",
      "88 4.267191410064697\n",
      "89 4.037667274475098\n",
      "90 3.821805953979492\n",
      "91 3.6178979873657227\n",
      "92 3.425684928894043\n",
      "93 3.2446978092193604\n",
      "94 3.074014186859131\n",
      "95 2.912937641143799\n",
      "96 2.760983467102051\n",
      "97 2.6177337169647217\n",
      "98 2.482624053955078\n",
      "99 2.3550801277160645\n",
      "100 2.2346577644348145\n",
      "101 2.120723247528076\n",
      "102 2.0131969451904297\n",
      "103 1.911362886428833\n",
      "104 1.814987301826477\n",
      "105 1.7238562107086182\n",
      "106 1.6375064849853516\n",
      "107 1.555582046508789\n",
      "108 1.47809898853302\n",
      "109 1.4048209190368652\n",
      "110 1.3353071212768555\n",
      "111 1.269533634185791\n",
      "112 1.2071213722229004\n",
      "113 1.1479592323303223\n",
      "114 1.0919826030731201\n",
      "115 1.0389119386672974\n",
      "116 0.9886678457260132\n",
      "117 0.9409630298614502\n",
      "118 0.8957129716873169\n",
      "119 0.852794885635376\n",
      "120 0.8120731711387634\n",
      "121 0.7735074162483215\n",
      "122 0.7368707656860352\n",
      "123 0.702059268951416\n",
      "124 0.6690755486488342\n",
      "125 0.6377242207527161\n",
      "126 0.6079617738723755\n",
      "127 0.5796899795532227\n",
      "128 0.5528005361557007\n",
      "129 0.5272296071052551\n",
      "130 0.5029308199882507\n",
      "131 0.47981125116348267\n",
      "132 0.45783740282058716\n",
      "133 0.43697497248649597\n",
      "134 0.41712871193885803\n",
      "135 0.39823561906814575\n",
      "136 0.3802500367164612\n",
      "137 0.3631371557712555\n",
      "138 0.3468387722969055\n",
      "139 0.3313336968421936\n",
      "140 0.3165729343891144\n",
      "141 0.30250823497772217\n",
      "142 0.28910642862319946\n",
      "143 0.27635082602500916\n",
      "144 0.26420459151268005\n",
      "145 0.25262022018432617\n",
      "146 0.2415725737810135\n",
      "147 0.23105134069919586\n",
      "148 0.22101525962352753\n",
      "149 0.21144318580627441\n",
      "150 0.20231309533119202\n",
      "151 0.19360098242759705\n",
      "152 0.1852981448173523\n",
      "153 0.17737558484077454\n",
      "154 0.16981014609336853\n",
      "155 0.16258585453033447\n",
      "156 0.15570005774497986\n",
      "157 0.1491156816482544\n",
      "158 0.14283257722854614\n",
      "159 0.13683238625526428\n",
      "160 0.13110311329364777\n",
      "161 0.12563282251358032\n",
      "162 0.1204126626253128\n",
      "163 0.11543217301368713\n",
      "164 0.11066934466362\n",
      "165 0.10611803084611893\n",
      "166 0.1017722338438034\n",
      "167 0.09761369228363037\n",
      "168 0.09363741427659988\n",
      "169 0.08983366191387177\n",
      "170 0.08619678020477295\n",
      "171 0.08271900564432144\n",
      "172 0.07939059287309647\n",
      "173 0.0762011855840683\n",
      "174 0.07314816117286682\n",
      "175 0.07022757083177567\n",
      "176 0.06743109226226807\n",
      "177 0.06475435197353363\n",
      "178 0.06219295784831047\n",
      "179 0.05973735451698303\n",
      "180 0.05738506466150284\n",
      "181 0.05513136461377144\n",
      "182 0.052970223128795624\n",
      "183 0.050900861620903015\n",
      "184 0.04891658574342728\n",
      "185 0.04701457545161247\n",
      "186 0.04519164189696312\n",
      "187 0.043443839997053146\n",
      "188 0.041767656803131104\n",
      "189 0.040160857141017914\n",
      "190 0.03861943259835243\n",
      "191 0.037142857909202576\n",
      "192 0.03572458028793335\n",
      "193 0.034363359212875366\n",
      "194 0.03305702656507492\n",
      "195 0.0318046398460865\n",
      "196 0.03060200810432434\n",
      "197 0.029447823762893677\n",
      "198 0.028340086340904236\n",
      "199 0.027276165783405304\n",
      "200 0.02625419944524765\n",
      "201 0.025272700935602188\n",
      "202 0.024330947548151016\n",
      "203 0.023426413536071777\n",
      "204 0.022557798773050308\n",
      "205 0.021723035722970963\n",
      "206 0.020920641720294952\n",
      "207 0.02014942467212677\n",
      "208 0.019409120082855225\n",
      "209 0.018697435036301613\n",
      "210 0.018013624474406242\n",
      "211 0.017355702817440033\n",
      "212 0.016723506152629852\n",
      "213 0.016115877777338028\n",
      "214 0.015530855394899845\n",
      "215 0.014968921430408955\n",
      "216 0.014428271912038326\n",
      "217 0.01390819065272808\n",
      "218 0.013407565653324127\n",
      "219 0.012926006689667702\n",
      "220 0.01246308907866478\n",
      "221 0.012017224915325642\n",
      "222 0.011593908071517944\n",
      "223 0.011188100092113018\n",
      "224 0.010797733440995216\n",
      "225 0.010422092862427235\n",
      "226 0.010060613974928856\n",
      "227 0.009712515398859978\n",
      "228 0.009377382695674896\n",
      "229 0.009054681286215782\n",
      "230 0.008744065649807453\n",
      "231 0.00844506360590458\n",
      "232 0.008156790398061275\n",
      "233 0.007879338227212429\n",
      "234 0.00761190103366971\n",
      "235 0.007354428060352802\n",
      "236 0.007106229197233915\n",
      "237 0.006867214106023312\n",
      "238 0.006636627484112978\n",
      "239 0.006414266303181648\n",
      "240 0.006199943833053112\n",
      "241 0.00599337462335825\n",
      "242 0.00579429604113102\n",
      "243 0.005602315999567509\n",
      "244 0.005417109001427889\n",
      "245 0.005238477140665054\n",
      "246 0.005066188983619213\n",
      "247 0.004899933002889156\n",
      "248 0.004739430733025074\n",
      "249 0.0045847417786717415\n",
      "250 0.004435411654412746\n",
      "251 0.004291378892958164\n",
      "252 0.004152259323745966\n",
      "253 0.0040181041695177555\n",
      "254 0.003888543229550123\n",
      "255 0.003763415152207017\n",
      "256 0.003642698284238577\n",
      "257 0.003525985637679696\n",
      "258 0.003413267433643341\n",
      "259 0.0033046286553144455\n",
      "260 0.0031996339093893766\n",
      "261 0.0030980962328612804\n",
      "262 0.0029999357648193836\n",
      "263 0.0029052523896098137\n",
      "264 0.002813745057210326\n",
      "265 0.00272530154325068\n",
      "266 0.0026398017071187496\n",
      "267 0.0025571798905730247\n",
      "268 0.0024773329496383667\n",
      "269 0.002400160301476717\n",
      "270 0.002325573703274131\n",
      "271 0.002253443468362093\n",
      "272 0.0021836573723703623\n",
      "273 0.0021162242628633976\n",
      "274 0.0020510016474872828\n",
      "275 0.001988012110814452\n",
      "276 0.001926987897604704\n",
      "277 0.0018679858185350895\n",
      "278 0.001810911693610251\n",
      "279 0.001755687058903277\n",
      "280 0.0017023072578012943\n",
      "281 0.0016506202518939972\n",
      "282 0.0016006188234314322\n",
      "283 0.0015521883033216\n",
      "284 0.0015053915558382869\n",
      "285 0.0014600749127566814\n",
      "286 0.001416194369085133\n",
      "287 0.0013737358385697007\n",
      "288 0.0013326129410415888\n",
      "289 0.0012928149662911892\n",
      "290 0.0012542933691293001\n",
      "291 0.0012169710826128721\n",
      "292 0.0011808723211288452\n",
      "293 0.0011458510998636484\n",
      "294 0.0011119564296677709\n",
      "295 0.0010791419772431254\n",
      "296 0.001047356054186821\n",
      "297 0.0010165710700675845\n",
      "298 0.0009867370827123523\n",
      "299 0.0009578088647685945\n",
      "300 0.0009298325749114156\n",
      "301 0.0009026936604641378\n",
      "302 0.0008763843216001987\n",
      "303 0.0008509338367730379\n",
      "304 0.0008262440096586943\n",
      "305 0.0008022967958822846\n",
      "306 0.0007791120442561805\n",
      "307 0.0007566108833998442\n",
      "308 0.0007348282379098237\n",
      "309 0.0007137057837098837\n",
      "310 0.0006932108663022518\n",
      "311 0.0006733289337716997\n",
      "312 0.0006540734320878983\n",
      "313 0.0006353941280394793\n",
      "314 0.0006172768189571798\n",
      "315 0.0005997090484015644\n",
      "316 0.0005826582200825214\n",
      "317 0.0005661300383508205\n",
      "318 0.0005501128034666181\n",
      "319 0.0005345549434423447\n",
      "320 0.0005194460973143578\n",
      "321 0.0005048323655501008\n",
      "322 0.0004906305694021285\n",
      "323 0.0004768432700075209\n",
      "324 0.00046346234739758074\n",
      "325 0.0004504883836489171\n",
      "326 0.000437905837316066\n",
      "327 0.000425696955062449\n",
      "328 0.0004138232325203717\n",
      "329 0.00040229957085102797\n",
      "330 0.0003911363601218909\n",
      "331 0.00038029701681807637\n",
      "332 0.00036976925912313163\n",
      "333 0.000359542784281075\n",
      "334 0.00034961168421432376\n",
      "335 0.00033996751881204545\n",
      "336 0.0003306203579995781\n",
      "337 0.00032152264611795545\n",
      "338 0.00031269746250472963\n",
      "339 0.0003041278978344053\n",
      "340 0.0002958119148388505\n",
      "341 0.00028772506630048156\n",
      "342 0.00027987745124846697\n",
      "343 0.0002722481731325388\n",
      "344 0.0002648388617672026\n",
      "345 0.0002576444239821285\n",
      "346 0.0002506510354578495\n",
      "347 0.000243860122282058\n",
      "348 0.00023726416111458093\n",
      "349 0.00023085169959813356\n",
      "350 0.00022462126798927784\n",
      "351 0.0002185752964578569\n",
      "352 0.00021268724231049418\n",
      "353 0.00020697456784546375\n",
      "354 0.00020141751156188548\n",
      "355 0.00019602273823693395\n",
      "356 0.00019078166224062443\n",
      "357 0.0001856765302363783\n",
      "358 0.00018071768863592297\n",
      "359 0.00017589658091310412\n",
      "360 0.00017121760174632072\n",
      "361 0.00016665412113070488\n",
      "362 0.00016223412239924073\n",
      "363 0.00015792244812473655\n",
      "364 0.00015373465430457145\n",
      "365 0.000149659434100613\n",
      "366 0.00014570522762369365\n",
      "367 0.00014185725012794137\n",
      "368 0.00013811267854180187\n",
      "369 0.00013447149831335992\n",
      "370 0.00013093877350911498\n",
      "371 0.00012749028974212706\n",
      "372 0.00012414735101629049\n",
      "373 0.00012089244410162792\n",
      "374 0.00011772750440286472\n",
      "375 0.0001146480135503225\n",
      "376 0.00011165167961735278\n",
      "377 0.00010872958955587819\n",
      "378 0.00010590362944640219\n",
      "379 0.0001031436986522749\n",
      "380 0.00010045541421277449\n",
      "381 9.784803114598617e-05\n",
      "382 9.530436364002526e-05\n",
      "383 9.28317749639973e-05\n",
      "384 9.042720193974674e-05\n",
      "385 8.808785059954971e-05\n",
      "386 8.580602298025042e-05\n",
      "387 8.359679486602545e-05\n",
      "388 8.1435646279715e-05\n",
      "389 7.933862070785835e-05\n",
      "390 7.729389471933246e-05\n",
      "391 7.530725270044059e-05\n",
      "392 7.336999260587618e-05\n",
      "393 7.1483947976958e-05\n",
      "394 6.965023931115866e-05\n",
      "395 6.786364247091115e-05\n",
      "396 6.612962170038372e-05\n",
      "397 6.443493475671858e-05\n",
      "398 6.279109220486134e-05\n",
      "399 6.118615419836715e-05\n",
      "400 5.96257668803446e-05\n",
      "401 5.810654329252429e-05\n",
      "402 5.662767216563225e-05\n",
      "403 5.518395482795313e-05\n",
      "404 5.378479545470327e-05\n",
      "405 5.241803592070937e-05\n",
      "406 5.108680488774553e-05\n",
      "407 4.9792339268606156e-05\n",
      "408 4.8529705964028835e-05\n",
      "409 4.7302572056651115e-05\n",
      "410 4.6109926188364625e-05\n",
      "411 4.494469612836838e-05\n",
      "412 4.3807231122627854e-05\n",
      "413 4.27020677307155e-05\n",
      "414 4.162568075116724e-05\n",
      "415 4.057873593410477e-05\n",
      "416 3.95580027543474e-05\n",
      "417 3.85613675462082e-05\n",
      "418 3.75926865672227e-05\n",
      "419 3.664779433165677e-05\n",
      "420 3.5729375667870045e-05\n",
      "421 3.4836921258829534e-05\n",
      "422 3.396306783542968e-05\n",
      "423 3.3111500670202076e-05\n",
      "424 3.228469722671434e-05\n",
      "425 3.147884490317665e-05\n",
      "426 3.069275771849789e-05\n",
      "427 2.992857844219543e-05\n",
      "428 2.918158315878827e-05\n",
      "429 2.8453619961510412e-05\n",
      "430 2.7745572879211977e-05\n",
      "431 2.705376027734019e-05\n",
      "432 2.638243677210994e-05\n",
      "433 2.5726749299792573e-05\n",
      "434 2.5089240807574242e-05\n",
      "435 2.4468252377118915e-05\n",
      "436 2.385879633948207e-05\n",
      "437 2.3269543817150407e-05\n",
      "438 2.269321703352034e-05\n",
      "439 2.2132404410513118e-05\n",
      "440 2.1584692149190232e-05\n",
      "441 2.1053010641480796e-05\n",
      "442 2.053049684036523e-05\n",
      "443 2.0024679542984813e-05\n",
      "444 1.952965249074623e-05\n",
      "445 1.9049779439228587e-05\n",
      "446 1.8579190509626642e-05\n",
      "447 1.8122060282621533e-05\n",
      "448 1.767766661942005e-05\n",
      "449 1.7241614841623232e-05\n",
      "450 1.6820578821352683e-05\n",
      "451 1.6406351278419606e-05\n",
      "452 1.6002395568648353e-05\n",
      "453 1.560981763759628e-05\n",
      "454 1.5229717064357828e-05\n",
      "455 1.4854559594823513e-05\n",
      "456 1.4492141417576931e-05\n",
      "457 1.4136006939224899e-05\n",
      "458 1.3791523088002577e-05\n",
      "459 1.345666532870382e-05\n",
      "460 1.3126324120094068e-05\n",
      "461 1.2805050573660992e-05\n",
      "462 1.2494330803747289e-05\n",
      "463 1.2188780601718463e-05\n",
      "464 1.1892198017449118e-05\n",
      "465 1.1602863196458202e-05\n",
      "466 1.1319258192088455e-05\n",
      "467 1.1044745406252332e-05\n",
      "468 1.0776857379823923e-05\n",
      "469 1.0516101610846817e-05\n",
      "470 1.0259818736813031e-05\n",
      "471 1.0010293408413418e-05\n",
      "472 9.767238225322217e-06\n",
      "473 9.530504030408338e-06\n",
      "474 9.298724762629718e-06\n",
      "475 9.072234206541907e-06\n",
      "476 8.85262943484122e-06\n",
      "477 8.639824955025688e-06\n",
      "478 8.429596164205577e-06\n",
      "479 8.226497811847366e-06\n",
      "480 8.027242074604146e-06\n",
      "481 7.834664756956045e-06\n",
      "482 7.645116056664847e-06\n",
      "483 7.4609811235859524e-06\n",
      "484 7.279959845618578e-06\n",
      "485 7.105674285412533e-06\n",
      "486 6.9343413997557946e-06\n",
      "487 6.765887974324869e-06\n",
      "488 6.603233487112448e-06\n",
      "489 6.444193786592223e-06\n",
      "490 6.290124929364538e-06\n",
      "491 6.138409844425041e-06\n",
      "492 5.991059424559353e-06\n",
      "493 5.847932698088698e-06\n",
      "494 5.706463525712024e-06\n",
      "495 5.570214852923527e-06\n",
      "496 5.437442723632557e-06\n",
      "497 5.306134880811442e-06\n",
      "498 5.1790707402687985e-06\n",
      "499 5.054607754573226e-06\n"
     ]
    }
   ],
   "source": [
    "# NN Version\n",
    "import torch\n",
    "\n",
    "# cpu or cuda\n",
    "device = torch.device('cpu')\n",
    "\n",
    "N, D_in, D_h, D_out = 100, 1000, 100, 10\n",
    "\n",
    "# generate training data\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# define the model\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, D_h),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(D_h, D_out),\n",
    ").to(device)\n",
    "\n",
    "# define the loss function\n",
    "loss_fn = torch.nn.MSELoss(reduction='sum')\n",
    "\n",
    "# training, use bigger learning rate\n",
    "learning_rate = 1e-4\n",
    "for t in range(500):\n",
    "    # forward inference\n",
    "    y_pred = model(x)\n",
    "    # calculate the loss\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "    # back-propagation\n",
    "    model.zero_grad()\n",
    "    loss.backward()\n",
    "    # update the model parameters(weights)\n",
    "    with torch.no_grad():\n",
    "        for param in model.parameters():\n",
    "            param -= learning_rate * param.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 1163.261962890625\n",
      "1 1077.5340576171875\n",
      "2 1005.70556640625\n",
      "3 943.58447265625\n",
      "4 888.6766357421875\n",
      "5 838.9414672851562\n",
      "6 793.19921875\n",
      "7 750.942626953125\n",
      "8 711.2891845703125\n",
      "9 673.778076171875\n",
      "10 638.21435546875\n",
      "11 604.2950439453125\n",
      "12 571.9231567382812\n",
      "13 540.854248046875\n",
      "14 511.00775146484375\n",
      "15 482.36749267578125\n",
      "16 454.7848815917969\n",
      "17 428.347412109375\n",
      "18 402.93359375\n",
      "19 378.5995178222656\n",
      "20 355.3414001464844\n",
      "21 333.22320556640625\n",
      "22 312.112060546875\n",
      "23 291.99652099609375\n",
      "24 272.922119140625\n",
      "25 254.904052734375\n",
      "26 237.93966674804688\n",
      "27 221.96177673339844\n",
      "28 206.92666625976562\n",
      "29 192.80487060546875\n",
      "30 179.54898071289062\n",
      "31 167.16567993164062\n",
      "32 155.61251831054688\n",
      "33 144.82412719726562\n",
      "34 134.79347229003906\n",
      "35 125.4600830078125\n",
      "36 116.77212524414062\n",
      "37 108.69270324707031\n",
      "38 101.19742584228516\n",
      "39 94.22679138183594\n",
      "40 87.76121520996094\n",
      "41 81.7593002319336\n",
      "42 76.1905746459961\n",
      "43 71.01890563964844\n",
      "44 66.228271484375\n",
      "45 61.783023834228516\n",
      "46 57.63160705566406\n",
      "47 53.77770233154297\n",
      "48 50.207191467285156\n",
      "49 46.89073944091797\n",
      "50 43.815162658691406\n",
      "51 40.961341857910156\n",
      "52 38.30497741699219\n",
      "53 35.83873748779297\n",
      "54 33.547157287597656\n",
      "55 31.41271209716797\n",
      "56 29.429874420166016\n",
      "57 27.587787628173828\n",
      "58 25.87326431274414\n",
      "59 24.274688720703125\n",
      "60 22.78785514831543\n",
      "61 21.401073455810547\n",
      "62 20.107646942138672\n",
      "63 18.900815963745117\n",
      "64 17.772541046142578\n",
      "65 16.718908309936523\n",
      "66 15.73468017578125\n",
      "67 14.813462257385254\n",
      "68 13.95005989074707\n",
      "69 13.142738342285156\n",
      "70 12.387055397033691\n",
      "71 11.678253173828125\n",
      "72 11.012102127075195\n",
      "73 10.388680458068848\n",
      "74 9.804461479187012\n",
      "75 9.255684852600098\n",
      "76 8.740534782409668\n",
      "77 8.255273818969727\n",
      "78 7.799940586090088\n",
      "79 7.372785568237305\n",
      "80 6.970139503479004\n",
      "81 6.59178352355957\n",
      "82 6.23563289642334\n",
      "83 5.900394439697266\n",
      "84 5.584249496459961\n",
      "85 5.286687850952148\n",
      "86 5.006389141082764\n",
      "87 4.742245197296143\n",
      "88 4.493001937866211\n",
      "89 4.2578654289245605\n",
      "90 4.036112308502197\n",
      "91 3.8269643783569336\n",
      "92 3.6292099952697754\n",
      "93 3.442007541656494\n",
      "94 3.26526141166687\n",
      "95 3.0980420112609863\n",
      "96 2.940001964569092\n",
      "97 2.7904603481292725\n",
      "98 2.649226188659668\n",
      "99 2.5160458087921143\n",
      "100 2.3897359371185303\n",
      "101 2.2701947689056396\n",
      "102 2.1569864749908447\n",
      "103 2.0497546195983887\n",
      "104 1.9482746124267578\n",
      "105 1.8521970510482788\n",
      "106 1.761150598526001\n",
      "107 1.6749398708343506\n",
      "108 1.593348741531372\n",
      "109 1.5160276889801025\n",
      "110 1.442509412765503\n",
      "111 1.372844934463501\n",
      "112 1.306715488433838\n",
      "113 1.2439125776290894\n",
      "114 1.184378743171692\n",
      "115 1.1279979944229126\n",
      "116 1.074420690536499\n",
      "117 1.0235549211502075\n",
      "118 0.9752285480499268\n",
      "119 0.9292478561401367\n",
      "120 0.8855401277542114\n",
      "121 0.8439875245094299\n",
      "122 0.8044834733009338\n",
      "123 0.7669957876205444\n",
      "124 0.7313616275787354\n",
      "125 0.6974681615829468\n",
      "126 0.6652188301086426\n",
      "127 0.634527325630188\n",
      "128 0.6053122282028198\n",
      "129 0.5775068402290344\n",
      "130 0.5510374307632446\n",
      "131 0.5258349180221558\n",
      "132 0.5018244981765747\n",
      "133 0.4789826273918152\n",
      "134 0.45722290873527527\n",
      "135 0.4364932179450989\n",
      "136 0.41675037145614624\n",
      "137 0.3979599177837372\n",
      "138 0.3800285756587982\n",
      "139 0.3629598617553711\n",
      "140 0.34668537974357605\n",
      "141 0.3311687707901001\n",
      "142 0.31635838747024536\n",
      "143 0.302245169878006\n",
      "144 0.28879159688949585\n",
      "145 0.27595752477645874\n",
      "146 0.2636917233467102\n",
      "147 0.25201696157455444\n",
      "148 0.24087534844875336\n",
      "149 0.23023787140846252\n",
      "150 0.22008375823497772\n",
      "151 0.21039435267448425\n",
      "152 0.20115947723388672\n",
      "153 0.19234757125377655\n",
      "154 0.18393951654434204\n",
      "155 0.17590594291687012\n",
      "156 0.1681879758834839\n",
      "157 0.16081911325454712\n",
      "158 0.15378275513648987\n",
      "159 0.1470678746700287\n",
      "160 0.14066441357135773\n",
      "161 0.13454394042491913\n",
      "162 0.1287011206150055\n",
      "163 0.12312501668930054\n",
      "164 0.11780089139938354\n",
      "165 0.11271321773529053\n",
      "166 0.10785886645317078\n",
      "167 0.1032133623957634\n",
      "168 0.09877561777830124\n",
      "169 0.09453245997428894\n",
      "170 0.09048048406839371\n",
      "171 0.08661198616027832\n",
      "172 0.08291719108819962\n",
      "173 0.07938268780708313\n",
      "174 0.07600106298923492\n",
      "175 0.07276707887649536\n",
      "176 0.06967934221029282\n",
      "177 0.06672579050064087\n",
      "178 0.063902348279953\n",
      "179 0.06120321899652481\n",
      "180 0.05862418934702873\n",
      "181 0.056157827377319336\n",
      "182 0.053794048726558685\n",
      "183 0.05153508856892586\n",
      "184 0.04937103018164635\n",
      "185 0.047303833067417145\n",
      "186 0.04532761126756668\n",
      "187 0.04343470185995102\n",
      "188 0.04162280261516571\n",
      "189 0.03988954424858093\n",
      "190 0.03823146969079971\n",
      "191 0.03664703294634819\n",
      "192 0.03512899577617645\n",
      "193 0.03367515653371811\n",
      "194 0.03228224813938141\n",
      "195 0.030949069187045097\n",
      "196 0.02967255562543869\n",
      "197 0.028451377525925636\n",
      "198 0.027281291782855988\n",
      "199 0.026160655543208122\n",
      "200 0.02508709765970707\n",
      "201 0.024058841168880463\n",
      "202 0.023074325174093246\n",
      "203 0.02213236503303051\n",
      "204 0.021229080855846405\n",
      "205 0.020364444702863693\n",
      "206 0.01953626051545143\n",
      "207 0.018742144107818604\n",
      "208 0.017981283366680145\n",
      "209 0.01725168339908123\n",
      "210 0.01655426435172558\n",
      "211 0.015887252986431122\n",
      "212 0.015247464179992676\n",
      "213 0.014635058119893074\n",
      "214 0.014047568663954735\n",
      "215 0.01348387822508812\n",
      "216 0.012943364679813385\n",
      "217 0.012425380758941174\n",
      "218 0.011928767897188663\n",
      "219 0.011452297680079937\n",
      "220 0.010995027609169483\n",
      "221 0.010556954890489578\n",
      "222 0.010136486031115055\n",
      "223 0.009733209386467934\n",
      "224 0.009346614591777325\n",
      "225 0.00897594541311264\n",
      "226 0.008620057255029678\n",
      "227 0.008278598077595234\n",
      "228 0.007951430976390839\n",
      "229 0.007637492381036282\n",
      "230 0.007335818372666836\n",
      "231 0.007046873215585947\n",
      "232 0.006769362837076187\n",
      "233 0.006502949632704258\n",
      "234 0.006247514858841896\n",
      "235 0.006002072244882584\n",
      "236 0.005766639485955238\n",
      "237 0.005540449172258377\n",
      "238 0.005323531571775675\n",
      "239 0.0051152827218174934\n",
      "240 0.004915210418403149\n",
      "241 0.0047231800854206085\n",
      "242 0.004538799170404673\n",
      "243 0.0043619852513074875\n",
      "244 0.004192147869616747\n",
      "245 0.004028908908367157\n",
      "246 0.0038724583573639393\n",
      "247 0.003722083056345582\n",
      "248 0.00357770430855453\n",
      "249 0.003438930958509445\n",
      "250 0.003305742284283042\n",
      "251 0.0031778833363205194\n",
      "252 0.0030550071969628334\n",
      "253 0.002937052631750703\n",
      "254 0.00282361195422709\n",
      "255 0.002714652568101883\n",
      "256 0.0026100396644324064\n",
      "257 0.0025095685850828886\n",
      "258 0.002412979956716299\n",
      "259 0.0023202504962682724\n",
      "260 0.002231205813586712\n",
      "261 0.0021455809473991394\n",
      "262 0.002063295803964138\n",
      "263 0.0019843322224915028\n",
      "264 0.0019084034720435739\n",
      "265 0.001835419679991901\n",
      "266 0.0017652460373938084\n",
      "267 0.0016978641506284475\n",
      "268 0.0016331032384186983\n",
      "269 0.0015708260471001267\n",
      "270 0.0015109977684915066\n",
      "271 0.0014534721849486232\n",
      "272 0.0013982439413666725\n",
      "273 0.0013450859114527702\n",
      "274 0.0012940324377268553\n",
      "275 0.0012448750203475356\n",
      "276 0.0011976445093750954\n",
      "277 0.0011522790882736444\n",
      "278 0.0011087005259469151\n",
      "279 0.0010668020695447922\n",
      "280 0.0010264579905197024\n",
      "281 0.0009876349940896034\n",
      "282 0.0009503865148872137\n",
      "283 0.000914615171495825\n",
      "284 0.0008801533840596676\n",
      "285 0.0008469774620607495\n",
      "286 0.0008151116780936718\n",
      "287 0.0007844928768463433\n",
      "288 0.0007550160516984761\n",
      "289 0.000726636266335845\n",
      "290 0.0006993574206717312\n",
      "291 0.0006731296889483929\n",
      "292 0.0006479166331700981\n",
      "293 0.0006236698827706277\n",
      "294 0.0006003314629197121\n",
      "295 0.0005778894992545247\n",
      "296 0.0005563033628277481\n",
      "297 0.000535538129042834\n",
      "298 0.0005155420512892306\n",
      "299 0.0004963231040164828\n",
      "300 0.00047781295143067837\n",
      "301 0.00046000874135643244\n",
      "302 0.00044288410572335124\n",
      "303 0.0004264184972271323\n",
      "304 0.00041058045462705195\n",
      "305 0.00039533674134872854\n",
      "306 0.00038064317777752876\n",
      "307 0.00036653323331847787\n",
      "308 0.0003529562964104116\n",
      "309 0.0003398723783902824\n",
      "310 0.0003272691392339766\n",
      "311 0.0003151741111651063\n",
      "312 0.00030352393514476717\n",
      "313 0.00029231555527076125\n",
      "314 0.0002815256593748927\n",
      "315 0.00027112747193314135\n",
      "316 0.00026112960767932236\n",
      "317 0.0002515036321710795\n",
      "318 0.00024223703076131642\n",
      "319 0.00023331926786340773\n",
      "320 0.00022472970886155963\n",
      "321 0.00021646638924721628\n",
      "322 0.00020850799046456814\n",
      "323 0.0002008483570534736\n",
      "324 0.00019347706984262913\n",
      "325 0.00018638570327311754\n",
      "326 0.0001795485441107303\n",
      "327 0.00017297681188210845\n",
      "328 0.00016663047426845878\n",
      "329 0.00016053591389209032\n",
      "330 0.00015466561308130622\n",
      "331 0.00014900887617841363\n",
      "332 0.0001435607991879806\n",
      "333 0.00013831908290740103\n",
      "334 0.00013326597400009632\n",
      "335 0.0001284110185224563\n",
      "336 0.0001237292744917795\n",
      "337 0.0001192197814816609\n",
      "338 0.00011488285235827789\n",
      "339 0.00011069745232816786\n",
      "340 0.0001066680415533483\n",
      "341 0.00010279010166414082\n",
      "342 9.905178740154952e-05\n",
      "343 9.545363718643785e-05\n",
      "344 9.199221676681191e-05\n",
      "345 8.86539273778908e-05\n",
      "346 8.543922012904659e-05\n",
      "347 8.233275002567098e-05\n",
      "348 7.935761095723137e-05\n",
      "349 7.647606253158301e-05\n",
      "350 7.371389801846817e-05\n",
      "351 7.105095573933795e-05\n",
      "352 6.848019256722182e-05\n",
      "353 6.599813787033781e-05\n",
      "354 6.361855048453435e-05\n",
      "355 6.132087582955137e-05\n",
      "356 5.910789695917629e-05\n",
      "357 5.6978464272106066e-05\n",
      "358 5.492484342539683e-05\n",
      "359 5.294657967169769e-05\n",
      "360 5.1038714445894584e-05\n",
      "361 4.9203852540813386e-05\n",
      "362 4.743313184008002e-05\n",
      "363 4.572774196276441e-05\n",
      "364 4.407943561091088e-05\n",
      "365 4.249845369486138e-05\n",
      "366 4.097610872122459e-05\n",
      "367 3.950441168854013e-05\n",
      "368 3.808717883657664e-05\n",
      "369 3.6718010960612446e-05\n",
      "370 3.540331817930564e-05\n",
      "371 3.4135413443436846e-05\n",
      "372 3.2915650081122294e-05\n",
      "373 3.173498043906875e-05\n",
      "374 3.0601146136177704e-05\n",
      "375 2.9504952181014232e-05\n",
      "376 2.8448892408050597e-05\n",
      "377 2.7435624360805377e-05\n",
      "378 2.645636413944885e-05\n",
      "379 2.5509421902825125e-05\n",
      "380 2.459825736877974e-05\n",
      "381 2.3718332158750854e-05\n",
      "382 2.2872467525303364e-05\n",
      "383 2.206235149060376e-05\n",
      "384 2.1273317543091252e-05\n",
      "385 2.051728188234847e-05\n",
      "386 1.978691216208972e-05\n",
      "387 1.9082508515566587e-05\n",
      "388 1.8405447917757556e-05\n",
      "389 1.775148484739475e-05\n",
      "390 1.7117385141318664e-05\n",
      "391 1.651027014304418e-05\n",
      "392 1.5925077605061233e-05\n",
      "393 1.53581549966475e-05\n",
      "394 1.4812807421549223e-05\n",
      "395 1.4287354133557528e-05\n",
      "396 1.3783910617348738e-05\n",
      "397 1.3294603377289604e-05\n",
      "398 1.2823968063457869e-05\n",
      "399 1.2368836905807257e-05\n",
      "400 1.1931039807677735e-05\n",
      "401 1.1507121598697267e-05\n",
      "402 1.1100291885668412e-05\n",
      "403 1.0709965863497928e-05\n",
      "404 1.0329519682272803e-05\n",
      "405 9.965187928173691e-06\n",
      "406 9.61434307100717e-06\n",
      "407 9.273962859879248e-06\n",
      "408 8.947647074819542e-06\n",
      "409 8.632299795863219e-06\n",
      "410 8.3274717326276e-06\n",
      "411 8.035263817873783e-06\n",
      "412 7.75098305894062e-06\n",
      "413 7.478893166990019e-06\n",
      "414 7.216209723992506e-06\n",
      "415 6.961757208046038e-06\n",
      "416 6.71700581733603e-06\n",
      "417 6.480806405306794e-06\n",
      "418 6.253150786506012e-06\n",
      "419 6.033789304638049e-06\n",
      "420 5.820713340654038e-06\n",
      "421 5.616894668492023e-06\n",
      "422 5.419501121650683e-06\n",
      "423 5.229435828368878e-06\n",
      "424 5.04614035889972e-06\n",
      "425 4.8688671085983515e-06\n",
      "426 4.699517376138829e-06\n",
      "427 4.53448410553392e-06\n",
      "428 4.374950549390633e-06\n",
      "429 4.221427843731362e-06\n",
      "430 4.074871412740322e-06\n",
      "431 3.931778337573633e-06\n",
      "432 3.793871428570128e-06\n",
      "433 3.662459675979335e-06\n",
      "434 3.534828920237487e-06\n",
      "435 3.411370016692672e-06\n",
      "436 3.2917744192673126e-06\n",
      "437 3.177731286996277e-06\n",
      "438 3.0665737540402915e-06\n",
      "439 2.959916173495003e-06\n",
      "440 2.8567906156240497e-06\n",
      "441 2.7565972686716123e-06\n",
      "442 2.6609391170495655e-06\n",
      "443 2.567540832387749e-06\n",
      "444 2.4777759790595155e-06\n",
      "445 2.3915365545690292e-06\n",
      "446 2.3084680833562743e-06\n",
      "447 2.228534413006855e-06\n",
      "448 2.1505522909137653e-06\n",
      "449 2.0753304852405563e-06\n",
      "450 2.0037075501022628e-06\n",
      "451 1.9337571757205296e-06\n",
      "452 1.8665550669538788e-06\n",
      "453 1.8018079117609886e-06\n",
      "454 1.7391392930221627e-06\n",
      "455 1.6784709941930487e-06\n",
      "456 1.6204935491259675e-06\n",
      "457 1.5641871868865564e-06\n",
      "458 1.5101727512956131e-06\n",
      "459 1.4576805824617622e-06\n",
      "460 1.4075826584303286e-06\n",
      "461 1.3590894241133356e-06\n",
      "462 1.312200083702919e-06\n",
      "463 1.2668506315094419e-06\n",
      "464 1.2226885246491292e-06\n",
      "465 1.180486492557975e-06\n",
      "466 1.139434743890888e-06\n",
      "467 1.1002098290191498e-06\n",
      "468 1.0620691455187625e-06\n",
      "469 1.02551598502032e-06\n",
      "470 9.898926691676024e-07\n",
      "471 9.557448947816738e-07\n",
      "472 9.22184824503347e-07\n",
      "473 8.902110835151689e-07\n",
      "474 8.595165468250343e-07\n",
      "475 8.302575906782295e-07\n",
      "476 8.016759238671511e-07\n",
      "477 7.737581881883671e-07\n",
      "478 7.47516025967343e-07\n",
      "479 7.221203759399941e-07\n",
      "480 6.970535082473361e-07\n",
      "481 6.735505735377956e-07\n",
      "482 6.506543286377564e-07\n",
      "483 6.282887170527829e-07\n",
      "484 6.067804747544869e-07\n",
      "485 5.855750373484625e-07\n",
      "486 5.656412440657732e-07\n",
      "487 5.46149294677889e-07\n",
      "488 5.271000986795116e-07\n",
      "489 5.08981258917629e-07\n",
      "490 4.915975182484544e-07\n",
      "491 4.748114577068918e-07\n",
      "492 4.5848059926356655e-07\n",
      "493 4.4244808350413223e-07\n",
      "494 4.273086915418389e-07\n",
      "495 4.1295950836683915e-07\n",
      "496 3.9863323308964027e-07\n",
      "497 3.848766425562644e-07\n",
      "498 3.715628906775237e-07\n",
      "499 3.5890536764782155e-07\n"
     ]
    }
   ],
   "source": [
    "# Custom Module Version\n",
    "import torch\n",
    "from my_nn import TwoLayerNet\n",
    "\n",
    "# cpu or cuda\n",
    "device = torch.device('cpu')\n",
    "\n",
    "N, D_in, D_h, D_out = 100, 1000, 100, 10\n",
    "\n",
    "# generate training data\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "# define the model\n",
    "model = TwoLayerNet(D_in, D_h, D_out).to(device)\n",
    "\n",
    "# define the loss function\n",
    "loss_fn = torch.nn.MSELoss(reduction='sum')\n",
    "\n",
    "# training, use bigger learning rate\n",
    "learning_rate = 1e-4\n",
    "for t in range(500):\n",
    "    # forward inference\n",
    "    y_pred = model(x)\n",
    "    # calculate the loss\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "    # back-propagation\n",
    "    model.zero_grad()\n",
    "    loss.backward()\n",
    "    # update the model parameters(weights)\n",
    "    with torch.no_grad():\n",
    "        for param in model.parameters():\n",
    "            param -= learning_rate * param.grad"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "archer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
